[llvm] [Attributor] Fix Load/Store offsets in case of multiple access bins when an allocation size is changed. (PR #72029)
Vidush Singhal via llvm-commits
llvm-commits at lists.llvm.org
Thu May 23 16:00:43 PDT 2024
https://github.com/vidsinghal updated https://github.com/llvm/llvm-project/pull/72029
>From 8ed28c44beb589e4a8c4368f4f68b06197443f1e Mon Sep 17 00:00:00 2001
From: vidsinghal <vidush.sl at gmail.com>
Date: Sun, 5 Nov 2023 22:30:42 -0500
Subject: [PATCH 1/3] [Attributor] Fix Load/Store Offsets if multiple bins are
present for a pointer allocation.
---
llvm/include/llvm/Transforms/IPO/Attributor.h | 5 +
.../Transforms/IPO/AttributorAttributes.cpp | 181 ++++++--
.../2008-02-01-ReturnAttrs.ll | 8 +-
.../ArgumentPromotion/X86/attributes.ll | 28 +-
.../X86/min-legal-vector-width.ll | 128 +++---
.../Attributor/ArgumentPromotion/alignment.ll | 6 +-
.../Attributor/ArgumentPromotion/alloca-as.ll | 10 +-
.../Attributor/ArgumentPromotion/array.ll | 26 +-
.../Attributor/ArgumentPromotion/attrs.ll | 22 +-
.../Attributor/ArgumentPromotion/basictest.ll | 2 +-
.../Attributor/ArgumentPromotion/byval-2.ll | 22 +-
.../Attributor/ArgumentPromotion/byval.ll | 38 +-
.../ArgumentPromotion/control-flow2.ll | 2 +-
.../Attributor/ArgumentPromotion/crash.ll | 2 +
.../Attributor/ArgumentPromotion/fp80.ll | 2 +-
.../Attributor/ArgumentPromotion/inalloca.ll | 28 +-
.../live_called_from_dead.ll | 10 +-
.../live_called_from_dead_2.ll | 12 +-
.../pr33641_remove_arg_dbgvalue.ll | 2 +-
.../Attributor/ArgumentPromotion/profile.ll | 6 +-
.../Attributor/ArgumentPromotion/sret.ll | 5 +-
.../IPConstantProp/openmp_parallel_for.ll | 16 +-
.../IPConstantProp/return-argument.ll | 4 +-
llvm/test/Transforms/Attributor/align.ll | 16 +-
llvm/test/Transforms/Attributor/allocator.ll | 165 ++++---
.../Attributor/call-simplify-pointer-info.ll | 72 ++-
llvm/test/Transforms/Attributor/callbacks.ll | 70 +--
.../Transforms/Attributor/heap_to_stack.ll | 10 +-
.../Attributor/heap_to_stack_gpu.ll | 10 +-
.../Transforms/Attributor/internal-noalias.ll | 14 +-
llvm/test/Transforms/Attributor/noalias.ll | 38 +-
.../test/Transforms/Attributor/nocapture-2.ll | 52 ++-
llvm/test/Transforms/Attributor/nofpclass.ll | 84 +++-
llvm/test/Transforms/Attributor/norecurse.ll | 6 +-
.../Transforms/Attributor/openmp_parallel.ll | 12 +-
.../Transforms/Attributor/pointer-info.ll | 8 +-
.../Attributor/value-simplify-assume.ll | 220 ++++-----
.../Attributor/value-simplify-dominance.ll | 24 +-
.../Attributor/value-simplify-gpu.ll | 6 +-
.../Attributor/value-simplify-local-remote.ll | 16 +-
.../Transforms/Attributor/value-simplify.ll | 30 +-
.../Transforms/OpenMP/parallel_deletion.ll | 67 ++-
.../OpenMP/parallel_region_merging.ll | 422 ++++++++++--------
43 files changed, 1149 insertions(+), 758 deletions(-)
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index 30c51250af61c..fb96e52121f82 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -6122,6 +6122,8 @@ struct AAPointerInfo : public AbstractAttribute {
virtual const_bin_iterator begin() const = 0;
virtual const_bin_iterator end() const = 0;
virtual int64_t numOffsetBins() const = 0;
+ virtual void dumpState(raw_ostream &O) const = 0;
+ virtual const Access &getBinAccess(unsigned Index) const = 0;
/// Call \p CB on all accesses that might interfere with \p Range and return
/// true if all such accesses were known and the callback returned true for
@@ -6293,6 +6295,9 @@ struct AAAllocationInfo : public StateWrapper<BooleanState, AbstractAttribute> {
virtual std::optional<TypeSize> getAllocatedSize() const = 0;
+ using NewOffsetsTy = DenseMap<AA::RangeTy, AA::RangeTy>;
+ virtual const NewOffsetsTy &getNewOffsets() const = 0;
+
/// See AbstractAttribute::getName()
const std::string getName() const override { return "AAAllocationInfo"; }
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 41b66aafe7d34..dc985261ca7c6 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -1083,6 +1083,10 @@ struct AAPointerInfoImpl
return State::numOffsetBins();
}
+ virtual const Access &getBinAccess(unsigned Index) const override {
+ return getAccess(Index);
+ }
+
bool forallInterferingAccesses(
AA::RangeTy Range,
function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
@@ -1429,7 +1433,7 @@ struct AAPointerInfoImpl
void trackPointerInfoStatistics(const IRPosition &IRP) const {}
/// Dump the state into \p O.
- void dumpState(raw_ostream &O) {
+ virtual void dumpState(raw_ostream &O) const override {
for (auto &It : OffsetBins) {
O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
<< "] : " << It.getSecond().size() << "\n";
@@ -12686,6 +12690,11 @@ struct AAAllocationInfoImpl : public AAAllocationInfo {
return AssumedAllocatedSize;
}
+ const NewOffsetsTy &getNewOffsets() const override {
+ assert(isValidState() && "the AA is invalid");
+ return NewComputedOffsets;
+ }
+
std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
const DataLayout &DL) {
@@ -12735,37 +12744,42 @@ struct AAAllocationInfoImpl : public AAAllocationInfo {
if (*AllocationSize == 0)
return indicatePessimisticFixpoint();
- int64_t BinSize = PI->numOffsetBins();
-
- // TODO: implement for multiple bins
- if (BinSize > 1)
- return indicatePessimisticFixpoint();
+ int64_t NumBins = PI->numOffsetBins();
- if (BinSize == 0) {
+ if (NumBins == 0) {
auto NewAllocationSize = std::optional<TypeSize>(TypeSize(0, false));
if (!changeAllocationSize(NewAllocationSize))
return ChangeStatus::UNCHANGED;
return ChangeStatus::CHANGED;
}
- // TODO: refactor this to be part of multiple bin case
- const auto &It = PI->begin();
+ // For each access bin
+ // Compute its new start Offset and store the results in a new map
+ // (NewOffsetBins).
+ int64_t PrevBinEndOffset = 0;
+ bool ChangedOffsets = false;
+ for (AAPointerInfo::OffsetBinsTy::const_iterator It = PI->begin();
+ It != PI->end(); It++) {
+ const AA::RangeTy &OldRange = It->getFirst();
+ int64_t NewStartOffset = PrevBinEndOffset;
+ int64_t NewEndOffset = NewStartOffset + OldRange.Size;
+ PrevBinEndOffset = NewEndOffset;
- // TODO: handle if Offset is not zero
- if (It->first.Offset != 0)
- return indicatePessimisticFixpoint();
-
- uint64_t SizeOfBin = It->first.Offset + It->first.Size;
-
- if (SizeOfBin >= *AllocationSize)
- return indicatePessimisticFixpoint();
+ ChangedOffsets |= setNewOffsets(OldRange, OldRange.Offset, NewStartOffset,
+ OldRange.Size);
+ }
+ // Set the new size of the allocation, the new size of the Allocation should
+ // be the size of NewEndOffset * 8, in bits.
auto NewAllocationSize =
- std::optional<TypeSize>(TypeSize(SizeOfBin * 8, false));
+ std::optional<TypeSize>(TypeSize(PrevBinEndOffset * 8, false));
if (!changeAllocationSize(NewAllocationSize))
return ChangeStatus::UNCHANGED;
+ if (!ChangedOffsets)
+ return ChangeStatus::UNCHANGED;
+
return ChangeStatus::CHANGED;
}
@@ -12775,12 +12789,13 @@ struct AAAllocationInfoImpl : public AAAllocationInfo {
assert(isValidState() &&
"Manifest should only be called if the state is valid.");
- Instruction *I = getIRPosition().getCtxI();
+ bool Changed = false;
+ const IRPosition &IRP = getIRPosition();
+ Instruction *I = IRP.getCtxI();
auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
-
switch (I->getOpcode()) {
// TODO: add case for malloc like calls
case Instruction::Alloca: {
@@ -12789,25 +12804,98 @@ struct AAAllocationInfoImpl : public AAAllocationInfo {
Type *CharType = Type::getInt8Ty(I->getContext());
- auto *NumBytesToValue =
- ConstantInt::get(I->getContext(), APInt(32, NumBytesToAllocate));
+ Type *CharArrayType = ArrayType::get(CharType, NumBytesToAllocate);
BasicBlock::iterator insertPt = AI->getIterator();
insertPt = std::next(insertPt);
- AllocaInst *NewAllocaInst =
- new AllocaInst(CharType, AI->getAddressSpace(), NumBytesToValue,
- AI->getAlign(), AI->getName(), insertPt);
-
- if (A.changeAfterManifest(IRPosition::inst(*AI), *NewAllocaInst))
- return ChangeStatus::CHANGED;
+ AllocaInst *NewAllocaInst = new AllocaInst(
+ CharArrayType, AI->getAddressSpace(), AI->getName(), insertPt);
+ Changed |= A.changeAfterManifest(IRPosition::inst(*AI), *NewAllocaInst);
break;
}
default:
break;
}
- return ChangeStatus::UNCHANGED;
+ const AAPointerInfo *PI =
+ A.getOrCreateAAFor<AAPointerInfo>(IRP, *this, DepClassTy::REQUIRED);
+
+ if (!PI)
+ return ChangeStatus::UNCHANGED;
+
+ if (!PI->getState().isValidState())
+ return ChangeStatus::UNCHANGED;
+
+ const auto &NewOffsetsMap = getNewOffsets();
+ for (AAPointerInfo::OffsetBinsTy::const_iterator It = PI->begin();
+ It != PI->end(); It++) {
+
+ const auto &OldOffsetRange = It->getFirst();
+
+ // If the OldOffsetRange is not in the map, offsets for that bin did not
+ // change We should just continue and skip changing the offsets in that
+ // case
+ if (!NewOffsetsMap.contains(OldOffsetRange))
+ continue;
+
+ const auto &NewOffsetRange = NewOffsetsMap.lookup(OldOffsetRange);
+ for (const auto AccIndex : It->getSecond()) {
+
+ const auto &AccessInstruction = PI->getBinAccess(AccIndex);
+ auto *LocalInst = AccessInstruction.getLocalInst();
+
+ switch (LocalInst->getOpcode()) {
+ case Instruction::Load: {
+ LoadInst *OldLoadInst = cast<LoadInst>(LocalInst);
+ Value *PointerOperand = OldLoadInst->getPointerOperand();
+
+ IntegerType *Int8TyInteger =
+ IntegerType::get(LocalInst->getContext(), 8);
+ IntegerType *Int64TyInteger =
+ IntegerType::get(LocalInst->getContext(), 64);
+ Value *indexList[2] = {
+ ConstantInt::get(Int64TyInteger, 0),
+ ConstantInt::get(Int64TyInteger,
+ NewOffsetRange.Offset - OldOffsetRange.Offset)};
+ Value *GepToNewAddress = GetElementPtrInst::Create(
+ Int8TyInteger, PointerOperand, indexList, "NewGep", OldLoadInst);
+
+ LoadInst *NewLoadInst =
+ new LoadInst(OldLoadInst->getType(), GepToNewAddress,
+ OldLoadInst->getName(), OldLoadInst);
+ Changed |= A.changeAfterManifest(IRPosition::inst(*OldLoadInst),
+ *NewLoadInst);
+ break;
+ }
+ case Instruction::Store: {
+ StoreInst *OldStoreInst = cast<StoreInst>(LocalInst);
+ Value *PointerOperand = OldStoreInst->getPointerOperand();
+
+ IntegerType *Int8TyInteger =
+ IntegerType::get(LocalInst->getContext(), 8);
+ IntegerType *Int64TyInteger =
+ IntegerType::get(LocalInst->getContext(), 64);
+ Value *indexList[2] = {
+ ConstantInt::get(Int64TyInteger, 0),
+ ConstantInt::get(Int64TyInteger,
+ NewOffsetRange.Offset - OldOffsetRange.Offset)};
+ Value *GepToNewAddress = GetElementPtrInst::Create(
+ Int8TyInteger, PointerOperand, indexList, "NewGep", OldStoreInst);
+
+ StoreInst *NewStoreInst = new StoreInst(
+ OldStoreInst->getValueOperand(), GepToNewAddress, OldStoreInst);
+ Changed |= A.changeAfterManifest(IRPosition::inst(*OldStoreInst),
+ *NewStoreInst);
+ break;
+ }
+ }
+ }
+ }
+
+ if (!Changed)
+ return ChangeStatus::UNCHANGED;
+ return ChangeStatus::CHANGED;
}
/// See AbstractAttribute::getAsStr().
@@ -12821,8 +12909,28 @@ struct AAAllocationInfoImpl : public AAAllocationInfo {
")";
}
+ void dumpNewOffsetBins(raw_ostream &O) {
+
+ O << "Printing Map from [OldOffsetsRange] : [NewOffsetsRange] if the "
+ "offsets changed."
+ << "\n";
+ const auto &NewOffsetsMap = getNewOffsets();
+ for (auto It = NewOffsetsMap.begin(); It != NewOffsetsMap.end(); It++) {
+
+ const auto &OldRange = It->getFirst();
+ const auto &NewRange = It->getSecond();
+
+ O << "[" << OldRange.Offset << "," << OldRange.Offset + OldRange.Size
+ << "] : ";
+ O << "[" << NewRange.Offset << "," << NewRange.Offset + NewRange.Size
+ << "]";
+ O << "\n";
+ }
+ }
+
private:
std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
+ NewOffsetsTy NewComputedOffsets;
// Maintain the computed allocation size of the object.
// Returns (bool) weather the size of the allocation was modified or not.
@@ -12834,6 +12942,21 @@ struct AAAllocationInfoImpl : public AAAllocationInfo {
}
return false;
}
+
+ // Maps an old byte range to its new Offset range in the new allocation.
+ // Returns (bool) weather the old byte range's offsets changed or not.
+ bool setNewOffsets(const AA::RangeTy &OldRange, int64_t OldOffset,
+ int64_t NewComputedOffset, int64_t Size) {
+
+ if (OldOffset == NewComputedOffset)
+ return false;
+
+ AA::RangeTy &NewRange = NewComputedOffsets.getOrInsertDefault(OldRange);
+ NewRange.Offset = NewComputedOffset;
+ NewRange.Size = Size;
+
+ return true;
+ }
};
struct AAAllocationInfoFloating : AAAllocationInfoImpl {
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
index adac17e62807c..e6062da1d1457 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/2008-02-01-ReturnAttrs.ll
@@ -22,16 +22,16 @@ define i32 @f(i32 %x) {
; TUNIT-LABEL: define {{[^@]+}}@f
; TUNIT-SAME: (i32 returned [[X:%.*]]) #[[ATTR0:[0-9]+]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[X_ADDR:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: store i32 [[X]], ptr [[X_ADDR]], align 4
+; TUNIT-NEXT: [[X_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: store i32 [[X]], ptr [[X_ADDR1]], align 4
; TUNIT-NEXT: ret i32 [[X]]
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define {{[^@]+}}@f
; CGSCC-SAME: (i32 [[X:%.*]]) #[[ATTR1:[0-9]+]] {
; CGSCC-NEXT: entry:
-; CGSCC-NEXT: [[X_ADDR:%.*]] = alloca i32, align 4
-; CGSCC-NEXT: store i32 [[X]], ptr [[X_ADDR]], align 4
+; CGSCC-NEXT: [[X_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CGSCC-NEXT: store i32 [[X]], ptr [[X_ADDR1]], align 4
; CGSCC-NEXT: [[TMP1:%.*]] = call i32 @deref(i32 [[X]]) #[[ATTR2:[0-9]+]]
; CGSCC-NEXT: ret i32 [[TMP1]]
;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
index 23415d3d3262b..7dfe6ef96a691 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/attributes.ll
@@ -26,11 +26,11 @@ define void @no_promote(ptr %arg) #1 {
; TUNIT-LABEL: define {{[^@]+}}@no_promote
; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca <4 x i64>, align 32
-; TUNIT-NEXT: [[TMP2:%.*]] = alloca <4 x i64>, align 32
-; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3:[0-9]+]]
-; TUNIT-NEXT: call fastcc void @no_promote_avx2(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP2]], ptr noalias nocapture nofree noundef nonnull readonly align 32 dereferenceable(32) [[TMP]]) #[[ATTR4:[0-9]+]]
-; TUNIT-NEXT: [[TMP4:%.*]] = load <4 x i64>, ptr [[TMP2]], align 32
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 32, align 32
+; TUNIT-NEXT: [[TMP22:%.*]] = alloca i8, i32 32, align 32
+; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3:[0-9]+]]
+; TUNIT-NEXT: call fastcc void @no_promote_avx2(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP22]], ptr noalias nocapture nofree noundef nonnull readonly align 32 dereferenceable(32) [[TMP1]]) #[[ATTR4:[0-9]+]]
+; TUNIT-NEXT: [[TMP4:%.*]] = load <4 x i64>, ptr [[TMP22]], align 32
; TUNIT-NEXT: store <4 x i64> [[TMP4]], ptr [[ARG]], align 2
; TUNIT-NEXT: ret void
;
@@ -78,12 +78,12 @@ define void @promote(ptr %arg) #0 {
; TUNIT-LABEL: define {{[^@]+}}@promote
; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR0]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca <4 x i64>, align 32
-; TUNIT-NEXT: [[TMP2:%.*]] = alloca <4 x i64>, align 32
-; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3]]
-; TUNIT-NEXT: [[TMP0:%.*]] = load <4 x i64>, ptr [[TMP]], align 32
-; TUNIT-NEXT: call fastcc void @promote_avx2(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP2]], <4 x i64> [[TMP0]]) #[[ATTR4]]
-; TUNIT-NEXT: [[TMP4:%.*]] = load <4 x i64>, ptr [[TMP2]], align 32
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 32, align 32
+; TUNIT-NEXT: [[TMP22:%.*]] = alloca i8, i32 32, align 32
+; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3]]
+; TUNIT-NEXT: [[TMP0:%.*]] = load <4 x i64>, ptr [[TMP1]], align 32
+; TUNIT-NEXT: call fastcc void @promote_avx2(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP22]], <4 x i64> [[TMP0]]) #[[ATTR4]]
+; TUNIT-NEXT: [[TMP4:%.*]] = load <4 x i64>, ptr [[TMP22]], align 32
; TUNIT-NEXT: store <4 x i64> [[TMP4]], ptr [[ARG]], align 2
; TUNIT-NEXT: ret void
;
@@ -91,10 +91,10 @@ define void @promote(ptr %arg) #0 {
; CGSCC-LABEL: define {{[^@]+}}@promote
; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(32) [[ARG:%.*]]) #[[ATTR0]] {
; CGSCC-NEXT: bb:
-; CGSCC-NEXT: [[TMP:%.*]] = alloca <4 x i64>, align 32
+; CGSCC-NEXT: [[TMP1:%.*]] = alloca i8, i32 32, align 32
; CGSCC-NEXT: [[TMP2:%.*]] = alloca <4 x i64>, align 32
-; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3]]
-; CGSCC-NEXT: [[TMP0:%.*]] = load <4 x i64>, ptr [[TMP]], align 32
+; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR3]]
+; CGSCC-NEXT: [[TMP0:%.*]] = load <4 x i64>, ptr [[TMP1]], align 32
; CGSCC-NEXT: call fastcc void @promote_avx2(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(32) [[TMP2]], <4 x i64> [[TMP0]]) #[[ATTR4]]
; CGSCC-NEXT: [[TMP4:%.*]] = load <4 x i64>, ptr [[TMP2]], align 32
; CGSCC-NEXT: store <4 x i64> [[TMP4]], ptr [[ARG]], align 2
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
index f0bcf68b6444b..2b755b53604f4 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/X86/min-legal-vector-width.ll
@@ -31,12 +31,12 @@ define void @avx512_legal512_prefer512_call_avx512_legal512_prefer512(ptr %arg)
; TUNIT-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer512
; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR0]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5:[0-9]+]]
-; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT: call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6:[0-9]+]]
-; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
+; TUNIT-NEXT: [[TMP22:%.*]] = alloca i8, i32 64, align 32
+; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5:[0-9]+]]
+; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
+; TUNIT-NEXT: call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP22]], <8 x i64> [[TMP0]]) #[[ATTR6:[0-9]+]]
+; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP22]], align 64
; TUNIT-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
; TUNIT-NEXT: ret void
;
@@ -44,10 +44,10 @@ define void @avx512_legal512_prefer512_call_avx512_legal512_prefer512(ptr %arg)
; CGSCC-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer512
; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR0]] {
; CGSCC-NEXT: bb:
-; CGSCC-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CGSCC-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
; CGSCC-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5:[0-9]+]]
-; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
+; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5:[0-9]+]]
+; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
; CGSCC-NEXT: call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer512(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6:[0-9]+]]
; CGSCC-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
; CGSCC-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
@@ -88,12 +88,12 @@ define void @avx512_legal512_prefer256_call_avx512_legal512_prefer256(ptr %arg)
; TUNIT-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer256
; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR1]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT: call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
-; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
+; TUNIT-NEXT: [[TMP22:%.*]] = alloca i8, i32 64, align 32
+; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
+; TUNIT-NEXT: call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP22]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP22]], align 64
; TUNIT-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
; TUNIT-NEXT: ret void
;
@@ -101,10 +101,10 @@ define void @avx512_legal512_prefer256_call_avx512_legal512_prefer256(ptr %arg)
; CGSCC-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer256
; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR1]] {
; CGSCC-NEXT: bb:
-; CGSCC-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CGSCC-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
; CGSCC-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
+; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
; CGSCC-NEXT: call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
; CGSCC-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
; CGSCC-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
@@ -145,12 +145,12 @@ define void @avx512_legal512_prefer512_call_avx512_legal512_prefer256(ptr %arg)
; TUNIT-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer256
; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR0]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT: call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
-; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
+; TUNIT-NEXT: [[TMP22:%.*]] = alloca i8, i32 64, align 32
+; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
+; TUNIT-NEXT: call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP22]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP22]], align 64
; TUNIT-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
; TUNIT-NEXT: ret void
;
@@ -158,10 +158,10 @@ define void @avx512_legal512_prefer512_call_avx512_legal512_prefer256(ptr %arg)
; CGSCC-LABEL: define {{[^@]+}}@avx512_legal512_prefer512_call_avx512_legal512_prefer256
; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR0]] {
; CGSCC-NEXT: bb:
-; CGSCC-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CGSCC-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
; CGSCC-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
+; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
; CGSCC-NEXT: call fastcc void @callee_avx512_legal512_prefer512_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
; CGSCC-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
; CGSCC-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
@@ -202,12 +202,12 @@ define void @avx512_legal512_prefer256_call_avx512_legal512_prefer512(ptr %arg)
; TUNIT-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer512
; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR1]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT: call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
-; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
+; TUNIT-NEXT: [[TMP22:%.*]] = alloca i8, i32 64, align 32
+; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
+; TUNIT-NEXT: call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP22]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP22]], align 64
; TUNIT-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
; TUNIT-NEXT: ret void
;
@@ -215,10 +215,10 @@ define void @avx512_legal512_prefer256_call_avx512_legal512_prefer512(ptr %arg)
; CGSCC-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal512_prefer512
; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR1]] {
; CGSCC-NEXT: bb:
-; CGSCC-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CGSCC-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
; CGSCC-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
+; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
; CGSCC-NEXT: call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal512_prefer512(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
; CGSCC-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
; CGSCC-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
@@ -257,11 +257,11 @@ define void @avx512_legal256_prefer256_call_avx512_legal512_prefer256(ptr %arg)
; TUNIT-LABEL: define {{[^@]+}}@avx512_legal256_prefer256_call_avx512_legal512_prefer256
; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR2:[0-9]+]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; TUNIT-NEXT: call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(64) [[TMP]]) #[[ATTR6]]
-; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
+; TUNIT-NEXT: [[TMP22:%.*]] = alloca i8, i32 64, align 32
+; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT: call fastcc void @callee_avx512_legal256_prefer256_call_avx512_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP22]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(64) [[TMP1]]) #[[ATTR6]]
+; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP22]], align 64
; TUNIT-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
; TUNIT-NEXT: ret void
;
@@ -310,11 +310,11 @@ define void @avx512_legal512_prefer256_call_avx512_legal256_prefer256(ptr %arg)
; TUNIT-LABEL: define {{[^@]+}}@avx512_legal512_prefer256_call_avx512_legal256_prefer256
; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR1]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; TUNIT-NEXT: call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(64) [[TMP]]) #[[ATTR6]]
-; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
+; TUNIT-NEXT: [[TMP22:%.*]] = alloca i8, i32 64, align 32
+; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT: call fastcc void @callee_avx512_legal512_prefer256_call_avx512_legal256_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP22]], ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(64) [[TMP1]]) #[[ATTR6]]
+; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP22]], align 64
; TUNIT-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
; TUNIT-NEXT: ret void
;
@@ -365,12 +365,12 @@ define void @avx2_legal256_prefer256_call_avx2_legal512_prefer256(ptr %arg) #4 {
; TUNIT-LABEL: define {{[^@]+}}@avx2_legal256_prefer256_call_avx2_legal512_prefer256
; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR3]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT: call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
-; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
+; TUNIT-NEXT: [[TMP22:%.*]] = alloca i8, i32 64, align 32
+; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
+; TUNIT-NEXT: call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP22]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP22]], align 64
; TUNIT-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
; TUNIT-NEXT: ret void
;
@@ -378,10 +378,10 @@ define void @avx2_legal256_prefer256_call_avx2_legal512_prefer256(ptr %arg) #4 {
; CGSCC-LABEL: define {{[^@]+}}@avx2_legal256_prefer256_call_avx2_legal512_prefer256
; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR3]] {
; CGSCC-NEXT: bb:
-; CGSCC-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CGSCC-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
; CGSCC-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
+; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
; CGSCC-NEXT: call fastcc void @callee_avx2_legal256_prefer256_call_avx2_legal512_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
; CGSCC-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
; CGSCC-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
@@ -422,12 +422,12 @@ define void @avx2_legal512_prefer256_call_avx2_legal256_prefer256(ptr %arg) #3 {
; TUNIT-LABEL: define {{[^@]+}}@avx2_legal512_prefer256_call_avx2_legal256_prefer256
; TUNIT-SAME: (ptr nocapture nofree writeonly [[ARG:%.*]]) #[[ATTR3]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
-; TUNIT-NEXT: call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
-; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
+; TUNIT-NEXT: [[TMP22:%.*]] = alloca i8, i32 64, align 32
+; TUNIT-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 32 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; TUNIT-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
+; TUNIT-NEXT: call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP22]], <8 x i64> [[TMP0]]) #[[ATTR6]]
+; TUNIT-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP22]], align 64
; TUNIT-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
; TUNIT-NEXT: ret void
;
@@ -435,10 +435,10 @@ define void @avx2_legal512_prefer256_call_avx2_legal256_prefer256(ptr %arg) #3 {
; CGSCC-LABEL: define {{[^@]+}}@avx2_legal512_prefer256_call_avx2_legal256_prefer256
; CGSCC-SAME: (ptr nocapture nofree noundef nonnull writeonly align 2 dereferenceable(64) [[ARG:%.*]]) #[[ATTR3]] {
; CGSCC-NEXT: bb:
-; CGSCC-NEXT: [[TMP:%.*]] = alloca <8 x i64>, align 32
+; CGSCC-NEXT: [[TMP1:%.*]] = alloca i8, i32 96, align 32
; CGSCC-NEXT: [[TMP2:%.*]] = alloca <8 x i64>, align 32
-; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
-; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP]], align 64
+; CGSCC-NEXT: call void @llvm.memset.p0.i64(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP1]], i8 noundef 0, i64 noundef 32, i1 noundef false) #[[ATTR5]]
+; CGSCC-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[TMP1]], align 64
; CGSCC-NEXT: call fastcc void @callee_avx2_legal512_prefer256_call_avx2_legal256_prefer256(ptr noalias nocapture nofree noundef nonnull writeonly align 64 dereferenceable(64) [[TMP2]], <8 x i64> [[TMP0]]) #[[ATTR6]]
; CGSCC-NEXT: [[TMP4:%.*]] = load <8 x i64>, ptr [[TMP2]], align 64
; CGSCC-NEXT: store <8 x i64> [[TMP4]], ptr [[ARG]], align 2
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
index 54a5b8c564077..541c3ddafa953 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/alignment.ll
@@ -9,8 +9,8 @@
define void @f() {
; TUNIT-LABEL: define {{[^@]+}}@f() {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[A:%.*]] = alloca i32, align 1
-; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[A]], align 1
+; TUNIT-NEXT: [[A1:%.*]] = alloca i8, i32 4, align 1
+; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[A1]], align 1
; TUNIT-NEXT: call void @g(i32 [[TMP0]])
; TUNIT-NEXT: ret void
;
@@ -92,7 +92,7 @@ define i32 @callercaller() {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@callercaller
; TUNIT-SAME: () #[[ATTR1:[0-9]+]] {
-; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 4
+; TUNIT-NEXT: [[B1:%.*]] = alloca i8, i32 4, align 4
; TUNIT-NEXT: ret i32 3
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/alloca-as.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/alloca-as.ll
index 9e2cd06d26ea9..633dabfc9be1d 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/alloca-as.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/alloca-as.ll
@@ -10,17 +10,17 @@ define i32 @bar(i32 %arg) {
; TUNIT-LABEL: define {{[^@]+}}@bar
; TUNIT-SAME: (i32 [[ARG:%.*]]) {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: store i32 [[ARG]], ptr [[STACK]], align 4
-; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[STACK]], align 4
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: store i32 [[ARG]], ptr [[STACK1]], align 4
+; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[STACK1]], align 4
; TUNIT-NEXT: [[CALL:%.*]] = call i32 @foo(i32 [[TMP0]])
; TUNIT-NEXT: ret i32 [[CALL]]
;
; CGSCC-LABEL: define {{[^@]+}}@bar
; CGSCC-SAME: (i32 [[ARG:%.*]]) {
; CGSCC-NEXT: entry:
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i32, align 4
-; CGSCC-NEXT: store i32 [[ARG]], ptr [[STACK]], align 4
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, i32 4, align 4
+; CGSCC-NEXT: store i32 [[ARG]], ptr [[STACK1]], align 4
; CGSCC-NEXT: [[CALL:%.*]] = call i32 @foo(i32 [[ARG]])
; CGSCC-NEXT: ret i32 [[CALL]]
;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/array.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/array.ll
index a52bbfbe1a346..e8ed3b0788105 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/array.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/array.ll
@@ -11,10 +11,10 @@ define void @caller() {
; TUNIT-NEXT: entry:
; TUNIT-NEXT: [[LEFT:%.*]] = alloca [3 x i32], align 4
; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[LEFT]], align 4
-; TUNIT-NEXT: [[LEFT_B4:%.*]] = getelementptr i8, ptr [[LEFT]], i64 4
-; TUNIT-NEXT: [[TMP1:%.*]] = load i32, ptr [[LEFT_B4]], align 4
-; TUNIT-NEXT: [[LEFT_B8:%.*]] = getelementptr i8, ptr [[LEFT]], i64 8
-; TUNIT-NEXT: [[TMP2:%.*]] = load i32, ptr [[LEFT_B8]], align 4
+; TUNIT-NEXT: [[LEFT_0_1:%.*]] = getelementptr [3 x i32], ptr [[LEFT]], i64 0, i64 1
+; TUNIT-NEXT: [[TMP1:%.*]] = load i32, ptr [[LEFT_0_1]], align 4
+; TUNIT-NEXT: [[LEFT_0_2:%.*]] = getelementptr [3 x i32], ptr [[LEFT]], i64 0, i64 2
+; TUNIT-NEXT: [[TMP2:%.*]] = load i32, ptr [[LEFT_0_2]], align 4
; TUNIT-NEXT: call void @callee(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]])
; TUNIT-NEXT: ret void
;
@@ -30,25 +30,19 @@ entry:
}
define internal void @callee(ptr noalias %arg) {
-; CHECK: Function Attrs: memory(readwrite, argmem: none)
; CHECK-LABEL: define {{[^@]+}}@callee
-; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-SAME: (i32 [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[ARG_PRIV:%.*]] = alloca [3 x i32], align 4
; CHECK-NEXT: store i32 [[TMP0]], ptr [[ARG_PRIV]], align 4
-; CHECK-NEXT: [[ARG_PRIV_B4:%.*]] = getelementptr i8, ptr [[ARG_PRIV]], i64 4
-; CHECK-NEXT: store i32 [[TMP1]], ptr [[ARG_PRIV_B4]], align 4
-; CHECK-NEXT: [[ARG_PRIV_B8:%.*]] = getelementptr i8, ptr [[ARG_PRIV]], i64 8
-; CHECK-NEXT: store i32 [[TMP2]], ptr [[ARG_PRIV_B8]], align 4
-; CHECK-NEXT: call void @use(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(12) [[ARG_PRIV]])
+; CHECK-NEXT: [[ARG_PRIV_0_1:%.*]] = getelementptr [3 x i32], ptr [[ARG_PRIV]], i64 0, i64 1
+; CHECK-NEXT: store i32 [[TMP1]], ptr [[ARG_PRIV_0_1]], align 4
+; CHECK-NEXT: [[ARG_PRIV_0_2:%.*]] = getelementptr [3 x i32], ptr [[ARG_PRIV]], i64 0, i64 2
+; CHECK-NEXT: store i32 [[TMP2]], ptr [[ARG_PRIV_0_2]], align 4
+; CHECK-NEXT: call void @use(ptr noalias nocapture noundef nonnull readonly align 4 dereferenceable(12) [[ARG_PRIV]])
; CHECK-NEXT: ret void
;
entry:
call void @use(ptr %arg)
ret void
}
-;.
-; TUNIT: attributes #[[ATTR0]] = { memory(readwrite, argmem: none) }
-;.
-; CGSCC: attributes #[[ATTR0]] = { memory(readwrite, argmem: none) }
-;.
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
index 9ce752aa95ee8..88ee234a1aa99 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/attrs.ll
@@ -17,9 +17,9 @@ define internal i32 @f(ptr byval(%struct.ss) %b, ptr byval(i32) %X, i32 %i) noun
; CHECK-NEXT: store i32 [[TMP0]], ptr [[B_PRIV]], align 4
; CHECK-NEXT: [[B_PRIV_B4:%.*]] = getelementptr i8, ptr [[B_PRIV]], i64 4
; CHECK-NEXT: store i64 [[TMP1]], ptr [[B_PRIV_B4]], align 4
-; CHECK-NEXT: [[VAL1:%.*]] = load i32, ptr [[B_PRIV]], align 8
-; CHECK-NEXT: [[VAL2:%.*]] = add i32 [[VAL1]], 1
-; CHECK-NEXT: store i32 [[VAL2]], ptr [[B_PRIV]], align 8
+; CHECK-NEXT: [[TMP1]] = load i32, ptr [[B_PRIV]], align 8
+; CHECK-NEXT: [[TMP2]] = add i32 [[TMP1]], 1
+; CHECK-NEXT: store i32 [[TMP2]], ptr [[B_PRIV]], align 8
; CHECK-NEXT: store i32 0, ptr [[X_PRIV]], align 4
; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[X_PRIV]], align 4
; CHECK-NEXT: [[A:%.*]] = add i32 [[L]], [[VAL2]]
@@ -44,12 +44,12 @@ define i32 @test(ptr %X) {
; TUNIT-LABEL: define {{[^@]+}}@test
; TUNIT-SAME: (ptr nocapture nofree nonnull readonly [[X:%.*]]) #[[ATTR1:[0-9]+]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 8
-; TUNIT-NEXT: store i32 1, ptr [[S]], align 8
-; TUNIT-NEXT: [[VAL4:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
-; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[S]], align 8
-; TUNIT-NEXT: [[S_B4:%.*]] = getelementptr i8, ptr [[S]], i64 4
-; TUNIT-NEXT: [[TMP1:%.*]] = load i64, ptr [[S_B4]], align 8
+; TUNIT-NEXT: [[S1:%.*]] = alloca i8, i32 12, align 8
+; TUNIT-NEXT: store i32 1, ptr [[S1]], align 8
+; TUNIT-NEXT: [[TMP4:%.*]] = getelementptr [[STRUCT_SS:%.*]], ptr [[S1]], i32 0, i32 1
+; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[S1]], align 8
+; TUNIT-NEXT: [[S1_B4:%.*]] = getelementptr i8, ptr [[S1]], i64 4
+; TUNIT-NEXT: [[TMP1:%.*]] = load i64, ptr [[S1_B4]], align 8
; TUNIT-NEXT: [[TMP2:%.*]] = load i32, ptr [[X]], align 4
; TUNIT-NEXT: [[C:%.*]] = call i32 @f(i32 [[TMP0]], i64 [[TMP1]], i32 [[TMP2]]) #[[ATTR2:[0-9]+]]
; TUNIT-NEXT: ret i32 [[C]]
@@ -58,8 +58,8 @@ define i32 @test(ptr %X) {
; CGSCC-LABEL: define {{[^@]+}}@test
; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[X:%.*]]) #[[ATTR1:[0-9]+]] {
; CGSCC-NEXT: entry:
-; CGSCC-NEXT: [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 8
-; CGSCC-NEXT: [[VAL4:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
+; CGSCC-NEXT: [[S1:%.*]] = alloca i8, i32 12, align 8
+; CGSCC-NEXT: [[TMP4:%.*]] = getelementptr [[STRUCT_SS:%.*]], ptr [[S1]], i32 0, i32 1
; CGSCC-NEXT: [[TMP0:%.*]] = load i32, ptr [[X]], align 4
; CGSCC-NEXT: [[C:%.*]] = call i32 @f(i32 noundef 1, i64 noundef 2, i32 [[TMP0]]) #[[ATTR2:[0-9]+]]
; CGSCC-NEXT: ret i32 [[C]]
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
index 7f8aeb59fd151..e6b21b9f4ab04 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/basictest.ll
@@ -39,7 +39,7 @@ define i32 @callercaller() {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@callercaller
; TUNIT-SAME: () #[[ATTR0:[0-9]+]] {
-; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 4
+; TUNIT-NEXT: [[B1:%.*]] = alloca i8, i32 4, align 4
; TUNIT-NEXT: ret i32 3
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
index 9f7acd579b274..f494ebc2dcd03 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval-2.ll
@@ -15,9 +15,9 @@ define internal void @f(ptr byval(%struct.ss) %b, ptr byval(i32) %X) nounwind
; CHECK-NEXT: store i32 [[TMP0]], ptr [[B_PRIV]], align 4
; CHECK-NEXT: [[B_PRIV_B4:%.*]] = getelementptr i8, ptr [[B_PRIV]], i64 4
; CHECK-NEXT: store i64 [[TMP1]], ptr [[B_PRIV_B4]], align 4
-; CHECK-NEXT: [[VAL1:%.*]] = load i32, ptr [[B_PRIV]], align 8
-; CHECK-NEXT: [[VAL2:%.*]] = add i32 [[VAL1]], 1
-; CHECK-NEXT: store i32 [[VAL2]], ptr [[B_PRIV]], align 8
+; CHECK-NEXT: [[TMP1]] = load i32, ptr [[B_PRIV]], align 8
+; CHECK-NEXT: [[TMP2]] = add i32 [[TMP1]], 1
+; CHECK-NEXT: store i32 [[TMP2]], ptr [[B_PRIV]], align 8
; CHECK-NEXT: store i32 0, ptr [[X_PRIV]], align 4
; CHECK-NEXT: ret void
;
@@ -36,12 +36,12 @@ define i32 @test(ptr %X) {
; TUNIT-LABEL: define {{[^@]+}}@test
; TUNIT-SAME: (ptr nocapture nofree nonnull readonly [[X:%.*]]) #[[ATTR1:[0-9]+]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 8
-; TUNIT-NEXT: store i32 1, ptr [[S]], align 8
-; TUNIT-NEXT: [[VAL4:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
-; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[S]], align 8
-; TUNIT-NEXT: [[S_B4:%.*]] = getelementptr i8, ptr [[S]], i64 4
-; TUNIT-NEXT: [[TMP1:%.*]] = load i64, ptr [[S_B4]], align 8
+; TUNIT-NEXT: [[S1:%.*]] = alloca i8, i32 12, align 8
+; TUNIT-NEXT: store i32 1, ptr [[S1]], align 8
+; TUNIT-NEXT: [[TMP4:%.*]] = getelementptr [[STRUCT_SS:%.*]], ptr [[S1]], i32 0, i32 1
+; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[S1]], align 8
+; TUNIT-NEXT: [[S1_B4:%.*]] = getelementptr i8, ptr [[S1]], i64 4
+; TUNIT-NEXT: [[TMP1:%.*]] = load i64, ptr [[S1_B4]], align 8
; TUNIT-NEXT: [[TMP2:%.*]] = load i32, ptr [[X]], align 4
; TUNIT-NEXT: call void @f(i32 [[TMP0]], i64 [[TMP1]], i32 [[TMP2]]) #[[ATTR2:[0-9]+]]
; TUNIT-NEXT: ret i32 0
@@ -50,8 +50,8 @@ define i32 @test(ptr %X) {
; CGSCC-LABEL: define {{[^@]+}}@test
; CGSCC-SAME: (ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[X:%.*]]) #[[ATTR1:[0-9]+]] {
; CGSCC-NEXT: entry:
-; CGSCC-NEXT: [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 8
-; CGSCC-NEXT: [[VAL4:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
+; CGSCC-NEXT: [[S1:%.*]] = alloca i8, i32 12, align 8
+; CGSCC-NEXT: [[TMP4:%.*]] = getelementptr [[STRUCT_SS:%.*]], ptr [[S1]], i32 0, i32 1
; CGSCC-NEXT: ret i32 0
;
entry:
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
index 621c6cf94313e..52474170ea5db 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/byval.ll
@@ -15,10 +15,10 @@ define internal i32 @f(ptr byval(%struct.ss) %b) nounwind {
; CHECK-NEXT: store i32 [[TMP0]], ptr [[B_PRIV]], align 4
; CHECK-NEXT: [[B_PRIV_B4:%.*]] = getelementptr i8, ptr [[B_PRIV]], i64 4
; CHECK-NEXT: store i64 [[TMP1]], ptr [[B_PRIV_B4]], align 4
-; CHECK-NEXT: [[VAL1:%.*]] = load i32, ptr [[B_PRIV]], align 8
-; CHECK-NEXT: [[VAL2:%.*]] = add i32 [[VAL1]], 1
-; CHECK-NEXT: store i32 [[VAL2]], ptr [[B_PRIV]], align 8
-; CHECK-NEXT: ret i32 [[VAL1]]
+; CHECK-NEXT: [[TMP1]] = load i32, ptr [[B_PRIV]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT: store i32 [[TMP2]], ptr [[B_PRIV]], align 8
+; CHECK-NEXT: ret i32 [[TMP1]]
;
entry:
%val1 = load i32, ptr %b, align 4
@@ -37,10 +37,10 @@ define internal i32 @g(ptr byval(%struct.ss) align 32 %b) nounwind {
; CHECK-NEXT: store i32 [[TMP0]], ptr [[B_PRIV]], align 4
; CHECK-NEXT: [[B_PRIV_B4:%.*]] = getelementptr i8, ptr [[B_PRIV]], i64 4
; CHECK-NEXT: store i64 [[TMP1]], ptr [[B_PRIV_B4]], align 4
-; CHECK-NEXT: [[VAL1:%.*]] = load i32, ptr [[B_PRIV]], align 32
-; CHECK-NEXT: [[VAL2:%.*]] = add i32 [[VAL1]], 1
-; CHECK-NEXT: store i32 [[VAL2]], ptr [[B_PRIV]], align 32
-; CHECK-NEXT: ret i32 [[VAL2]]
+; CHECK-NEXT: [[TMP1]] = load i32, ptr [[B_PRIV]], align 32
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT: store i32 [[TMP2]], ptr [[B_PRIV]], align 32
+; CHECK-NEXT: ret i32 [[TMP2]]
;
entry:
%val1 = load i32, ptr %b, align 4
@@ -55,16 +55,16 @@ define i32 @main() nounwind {
; TUNIT-LABEL: define {{[^@]+}}@main
; TUNIT-SAME: () #[[ATTR0]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
-; TUNIT-NEXT: store i32 1, ptr [[S]], align 32
-; TUNIT-NEXT: [[VAL4:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
-; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[S]], align 8
-; TUNIT-NEXT: [[S_B41:%.*]] = getelementptr i8, ptr [[S]], i64 4
-; TUNIT-NEXT: [[TMP1:%.*]] = load i64, ptr [[S_B41]], align 8
+; TUNIT-NEXT: [[S1:%.*]] = alloca i8, i32 12, align 4
+; TUNIT-NEXT: store i32 1, ptr [[S1]], align 32
+; TUNIT-NEXT: [[TMP4:%.*]] = getelementptr [[STRUCT_SS:%.*]], ptr [[S1]], i32 0, i32 1
+; TUNIT-NEXT: [[TMP0:%.*]] = load i32, ptr [[S1]], align 8
+; TUNIT-NEXT: [[S1_B42:%.*]] = getelementptr i8, ptr [[S1]], i64 4
+; TUNIT-NEXT: [[TMP1:%.*]] = load i64, ptr [[S1_B42]], align 8
; TUNIT-NEXT: [[C0:%.*]] = call i32 @f(i32 [[TMP0]], i64 [[TMP1]]) #[[ATTR1:[0-9]+]]
-; TUNIT-NEXT: [[TMP2:%.*]] = load i32, ptr [[S]], align 32
-; TUNIT-NEXT: [[S_B4:%.*]] = getelementptr i8, ptr [[S]], i64 4
-; TUNIT-NEXT: [[TMP3:%.*]] = load i64, ptr [[S_B4]], align 32
+; TUNIT-NEXT: [[TMP2:%.*]] = load i32, ptr [[S1]], align 32
+; TUNIT-NEXT: [[S1_B4:%.*]] = getelementptr i8, ptr [[S1]], i64 4
+; TUNIT-NEXT: [[TMP3:%.*]] = load i64, ptr [[S1_B4]], align 32
; TUNIT-NEXT: [[C1:%.*]] = call i32 @g(i32 [[TMP2]], i64 [[TMP3]]) #[[ATTR1]]
; TUNIT-NEXT: [[A:%.*]] = add i32 [[C0]], [[C1]]
; TUNIT-NEXT: ret i32 [[A]]
@@ -73,8 +73,8 @@ define i32 @main() nounwind {
; CGSCC-LABEL: define {{[^@]+}}@main
; CGSCC-SAME: () #[[ATTR1:[0-9]+]] {
; CGSCC-NEXT: entry:
-; CGSCC-NEXT: [[S:%.*]] = alloca [[STRUCT_SS:%.*]], align 4
-; CGSCC-NEXT: [[VAL4:%.*]] = getelementptr [[STRUCT_SS]], ptr [[S]], i32 0, i32 1
+; CGSCC-NEXT: [[S2:%.*]] = alloca i8, i32 12, align 4
+; CGSCC-NEXT: [[TMP4:%.*]] = getelementptr [[STRUCT_SS:%.*]], ptr [[S2]], i32 0, i32 1
; CGSCC-NEXT: [[C0:%.*]] = call i32 @f(i32 noundef 1, i64 noundef 2) #[[ATTR2:[0-9]+]]
; CGSCC-NEXT: [[C1:%.*]] = call i32 @g(i32 noundef 1, i64 noundef 2) #[[ATTR2]]
; CGSCC-NEXT: [[A:%.*]] = add i32 [[C0]], [[C1]]
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
index ebeb24d374728..08bb7c68ca1c1 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/control-flow2.ll
@@ -31,7 +31,7 @@ define i32 @foo() {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@foo
; TUNIT-SAME: () #[[ATTR0:[0-9]+]] {
-; TUNIT-NEXT: [[A:%.*]] = alloca i32, align 4
+; TUNIT-NEXT: [[A1:%.*]] = alloca i8, i32 4, align 4
; TUNIT-NEXT: ret i32 17
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
index 595cb37c6c93e..2ad8ef403d88c 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/crash.ll
@@ -100,6 +100,8 @@ define i32 @test_inf_promote_caller(i32 %arg) {
; TUNIT-LABEL: define {{[^@]+}}@test_inf_promote_caller
; TUNIT-SAME: (i32 [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
; TUNIT-NEXT: bb:
+; TUNIT-NEXT: [[TMP3:%.*]] = alloca i8, i32 8, align 8
+; TUNIT-NEXT: [[TMP14:%.*]] = alloca i8, i32 8, align 8
; TUNIT-NEXT: ret i32 0
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
index 274181fa8b9ef..101850f464c41 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/fp80.ll
@@ -84,7 +84,7 @@ define internal i64 @CaptureAStruct(ptr byval(%struct.Foo) %a) {
; CGSCC-NEXT: store i32 [[TMP0]], ptr [[A_PRIV]], align 4
; CGSCC-NEXT: [[A_PRIV_B8:%.*]] = getelementptr i8, ptr [[A_PRIV]], i64 8
; CGSCC-NEXT: store i64 [[TMP1]], ptr [[A_PRIV_B8]], align 8
-; CGSCC-NEXT: [[A_PTR:%.*]] = alloca ptr, align 8
+; CGSCC-NEXT: [[A_PTR1:%.*]] = alloca i8, i32 8, align 8
; CGSCC-NEXT: br label [[LOOP:%.*]]
; CGSCC: loop:
; CGSCC-NEXT: [[PHI:%.*]] = phi ptr [ null, [[ENTRY:%.*]] ], [ [[A_PRIV]], [[LOOP]] ]
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
index 957a26ff7176b..fd16e665dba42 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/inalloca.ll
@@ -8,7 +8,7 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
; Argpromote + sroa should change this to passing the two integers by value.
define internal i32 @f(ptr inalloca(%struct.ss) %s) {
-; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read)
+; CHECK: Function Attrs: nofree norecurse nosync nounwind willreturn memory(argmem: read)
; CHECK-LABEL: define {{[^@]+}}@f
; CHECK-SAME: (ptr noalias nocapture nofree noundef nonnull inalloca([[STRUCT_SS:%.*]]) align 4 dereferenceable(8) [[S:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -27,7 +27,7 @@ entry:
}
define i32 @main() {
-; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; TUNIT: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@main
; TUNIT-SAME: () #[[ATTR1:[0-9]+]] {
; TUNIT-NEXT: entry:
@@ -38,7 +38,7 @@ define i32 @main() {
; TUNIT-NEXT: [[R:%.*]] = call i32 @f(ptr noalias nocapture nofree noundef nonnull inalloca([[STRUCT_SS]]) align 4 dereferenceable(8) [[S]]) #[[ATTR2:[0-9]+]]
; TUNIT-NEXT: ret i32 [[R]]
;
-; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
+; CGSCC: Function Attrs: nofree nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define {{[^@]+}}@main
; CGSCC-SAME: () #[[ATTR1:[0-9]+]] {
; CGSCC-NEXT: entry:
@@ -60,9 +60,9 @@ entry:
; Argpromote can't promote %a because of the icmp use.
define internal i1 @g(ptr %a, ptr inalloca(%struct.ss) %b) nounwind {
-; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; CGSCC: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define {{[^@]+}}@g
-; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull readnone align 4 dereferenceable(8) [[A:%.*]], ptr noalias nocapture nofree noundef nonnull writeonly inalloca([[STRUCT_SS:%.*]]) align 4 dereferenceable(8) [[B:%.*]]) #[[ATTR2:[0-9]+]] {
+; CGSCC-SAME: (ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(8) [[A:%.*]], ptr noalias nocapture nofree nonnull writeonly inalloca([[STRUCT_SS:%.*]]) align 4 dereferenceable(8) [[B:%.*]]) #[[ATTR2:[0-9]+]] {
; CGSCC-NEXT: entry:
; CGSCC-NEXT: ret i1 undef
;
@@ -72,13 +72,13 @@ entry:
}
define i32 @test() {
-; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
+; TUNIT: Function Attrs: nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@test
; TUNIT-SAME: () #[[ATTR1]] {
; TUNIT-NEXT: entry:
; TUNIT-NEXT: ret i32 0
;
-; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
+; CGSCC: Function Attrs: nofree nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define {{[^@]+}}@test
; CGSCC-SAME: () #[[ATTR1]] {
; CGSCC-NEXT: entry:
@@ -90,12 +90,12 @@ entry:
ret i32 0
}
;.
-; TUNIT: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read) }
-; TUNIT: attributes #[[ATTR1]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
-; TUNIT: attributes #[[ATTR2]] = { nofree nosync nounwind willreturn memory(read) }
+; TUNIT: attributes #[[ATTR0]] = { nofree norecurse nosync nounwind willreturn memory(argmem: read) }
+; TUNIT: attributes #[[ATTR1]] = { nofree norecurse nosync nounwind willreturn memory(none) }
+; TUNIT: attributes #[[ATTR2]] = { nofree nosync nounwind willreturn }
;.
-; CGSCC: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read) }
-; CGSCC: attributes #[[ATTR1]] = { mustprogress nofree nosync nounwind willreturn memory(none) }
-; CGSCC: attributes #[[ATTR2]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
-; CGSCC: attributes #[[ATTR3]] = { nofree willreturn memory(read) }
+; CGSCC: attributes #[[ATTR0]] = { nofree norecurse nosync nounwind willreturn memory(argmem: read) }
+; CGSCC: attributes #[[ATTR1]] = { nofree nosync nounwind willreturn memory(none) }
+; CGSCC: attributes #[[ATTR2]] = { nofree norecurse nosync nounwind willreturn memory(none) }
+; CGSCC: attributes #[[ATTR3]] = { willreturn }
;.
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
index 2df81d6cb1832..b3704c203d896 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead.ll
@@ -36,9 +36,9 @@ define internal i32 @caller(ptr %B) {
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define {{[^@]+}}@caller
; CGSCC-SAME: () #[[ATTR0]] {
-; CGSCC-NEXT: [[A:%.*]] = alloca i32, align 4
-; CGSCC-NEXT: [[A2:%.*]] = alloca i8, i32 0, align 4
-; CGSCC-NEXT: [[A1:%.*]] = alloca i8, i32 0, align 4
+; CGSCC-NEXT: [[A1:%.*]] = alloca i8, i32 4, align 4
+; CGSCC-NEXT: [[A12:%.*]] = alloca i8, i32 0, align 4
+; CGSCC-NEXT: [[A11:%.*]] = alloca i8, i32 0, align 4
; CGSCC-NEXT: ret i32 0
;
%A = alloca i32
@@ -51,13 +51,13 @@ define i32 @callercaller() {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@callercaller
; TUNIT-SAME: () #[[ATTR0:[0-9]+]] {
-; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 4
+; TUNIT-NEXT: [[B1:%.*]] = alloca i8, i32 4, align 4
; TUNIT-NEXT: ret i32 0
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define {{[^@]+}}@callercaller
; CGSCC-SAME: () #[[ATTR1:[0-9]+]] {
-; CGSCC-NEXT: [[B:%.*]] = alloca i32, align 4
+; CGSCC-NEXT: [[B1:%.*]] = alloca i8, i32 4, align 4
; CGSCC-NEXT: [[X:%.*]] = call noundef i32 @caller() #[[ATTR2:[0-9]+]]
; CGSCC-NEXT: ret i32 [[X]]
;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
index 7c28de24beea2..716e5f426bbc8 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/live_called_from_dead_2.ll
@@ -46,16 +46,16 @@ define internal i32 @caller(ptr %B) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
; TUNIT-LABEL: define {{[^@]+}}@caller
; TUNIT-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
-; TUNIT-NEXT: [[A:%.*]] = alloca i32, align 4
+; TUNIT-NEXT: [[A1:%.*]] = alloca i8, i32 4, align 4
; TUNIT-NEXT: [[C:%.*]] = call i32 @test(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR2:[0-9]+]]
; TUNIT-NEXT: ret i32 undef
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: write)
; CGSCC-LABEL: define {{[^@]+}}@caller
; CGSCC-SAME: (ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B:%.*]]) #[[ATTR0]] {
-; CGSCC-NEXT: [[A:%.*]] = alloca i32, align 4
-; CGSCC-NEXT: [[A2:%.*]] = alloca i8, i32 0, align 4
-; CGSCC-NEXT: [[A1:%.*]] = alloca i8, i32 0, align 4
+; CGSCC-NEXT: [[A1:%.*]] = alloca i8, i32 4, align 4
+; CGSCC-NEXT: [[A13:%.*]] = alloca i8, i32 0, align 4
+; CGSCC-NEXT: [[A12:%.*]] = alloca i8, i32 0, align 4
; CGSCC-NEXT: [[C:%.*]] = call i32 @test(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR2:[0-9]+]]
; CGSCC-NEXT: ret i32 0
;
@@ -69,8 +69,8 @@ define i32 @callercaller() {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@callercaller
; TUNIT-SAME: () #[[ATTR1:[0-9]+]] {
-; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: [[X:%.*]] = call i32 @caller(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR2]]
+; TUNIT-NEXT: [[B1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: [[X:%.*]] = call i32 @caller(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B1]]) #[[ATTR2]]
; TUNIT-NEXT: ret i32 0
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
index 8c02ca4e86608..4a9d8694a1e82 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/pr33641_remove_arg_dbgvalue.ll
@@ -18,7 +18,7 @@ define void @foo() {
; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; CHECK-LABEL: define {{[^@]+}}@foo
; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP:%.*]] = alloca ptr, align 8
+; CHECK-NEXT: [[TMP1:%.*]] = alloca i8, i32 8, align 8
; CHECK-NEXT: ret void
;
%tmp = alloca %fun_t
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
index 4074fcb743232..506119661e86a 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/profile.ll
@@ -7,9 +7,9 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
define void @caller() #0 {
; TUNIT-LABEL: define {{[^@]+}}@caller() {
-; TUNIT-NEXT: [[X:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: store i32 42, ptr [[X]], align 4
-; TUNIT-NEXT: [[TMP1:%.*]] = load i32, ptr [[X]], align 4
+; TUNIT-NEXT: [[X1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: store i32 42, ptr [[X1]], align 4
+; TUNIT-NEXT: [[TMP1:%.*]] = load i32, ptr [[X1]], align 4
; TUNIT-NEXT: call void @promote_i32_ptr(i32 [[TMP1]]), !prof [[PROF0:![0-9]+]]
; TUNIT-NEXT: ret void
;
diff --git a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
index cb5b01750aaf0..39de5947d3411 100644
--- a/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
+++ b/llvm/test/Transforms/Attributor/ArgumentPromotion/sret.ll
@@ -36,8 +36,9 @@ define void @f() {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@f
; TUNIT-SAME: () #[[ATTR1:[0-9]+]] {
-; TUNIT-NEXT: [[R:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: call void @add(ptr noalias nocapture nofree nonnull readnone align 8 dereferenceable(8) undef, ptr noalias nocapture nofree noundef nonnull writeonly sret(i32) align 4 dereferenceable(4) [[R]]) #[[ATTR2:[0-9]+]]
+; TUNIT-NEXT: [[R1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: [[PAIR2:%.*]] = alloca i8, i32 8, align 8
+; TUNIT-NEXT: call void @add(ptr noalias nocapture nofree nonnull readnone align 8 dereferenceable(8) undef, ptr noalias nocapture nofree noundef nonnull writeonly sret(i32) align 4 dereferenceable(4) [[R1]]) #[[ATTR2:[0-9]+]]
; TUNIT-NEXT: ret void
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
index 683da42b64811..ef1b961c13652 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/openmp_parallel_for.ll
@@ -34,8 +34,8 @@ define dso_local void @foo(i32 %N) {
; TUNIT-LABEL: define {{[^@]+}}@foo
; TUNIT-SAME: (i32 [[N:%.*]]) {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[N_ADDR:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: [[P:%.*]] = alloca float, align 4
+; TUNIT-NEXT: [[N_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: [[P2:%.*]] = alloca i8, i32 4, align 4
; TUNIT-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(4) undef, ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(4) undef, i64 undef)
; TUNIT-NEXT: ret void
;
@@ -64,12 +64,12 @@ define internal void @.omp_outlined.(ptr noalias %.global_tid., ptr noalias %.bo
; TUNIT-LABEL: define {{[^@]+}}@.omp_outlined.
; TUNIT-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr noalias nocapture nofree noundef nonnull readnone align 4 dereferenceable(4) [[N:%.*]], ptr noalias nocapture nofree noundef nonnull readnone align 4 dereferenceable(4) [[P:%.*]], i64 [[Q:%.*]]) {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[Q_ADDR:%.*]] = alloca i64, align 8
+; TUNIT-NEXT: [[Q_ADDR1:%.*]] = alloca i8, i32 8, align 8
; TUNIT-NEXT: [[DOTOMP_LB:%.*]] = alloca i32, align 4
; TUNIT-NEXT: [[DOTOMP_UB:%.*]] = alloca i32, align 4
; TUNIT-NEXT: [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
; TUNIT-NEXT: [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: store i64 4617315517961601024, ptr [[Q_ADDR]], align 8
+; TUNIT-NEXT: store i64 4617315517961601024, ptr [[Q_ADDR1]], align 8
; TUNIT-NEXT: br label [[OMP_PRECOND_THEN:%.*]]
; TUNIT: omp.precond.then:
; TUNIT-NEXT: store i32 0, ptr [[DOTOMP_LB]], align 4
@@ -100,7 +100,7 @@ define internal void @.omp_outlined.(ptr noalias %.global_tid., ptr noalias %.bo
; TUNIT-NEXT: br label [[OMP_INNER_FOR_END:%.*]]
; TUNIT: omp.inner.for.body:
; TUNIT-NEXT: [[ADD10:%.*]] = add nsw i32 [[DOTOMP_IV_0]], 2
-; TUNIT-NEXT: [[TMP11:%.*]] = load double, ptr [[Q_ADDR]], align 8
+; TUNIT-NEXT: [[TMP11:%.*]] = load double, ptr [[Q_ADDR1]], align 8
; TUNIT-NEXT: call void @bar(i32 [[ADD10]], float nofpclass(nan inf zero sub nnorm) 3.000000e+00, double [[TMP11]])
; TUNIT-NEXT: br label [[OMP_BODY_CONTINUE:%.*]]
; TUNIT: omp.body.continue:
@@ -120,12 +120,12 @@ define internal void @.omp_outlined.(ptr noalias %.global_tid., ptr noalias %.bo
; CGSCC-LABEL: define {{[^@]+}}@.omp_outlined.
; CGSCC-SAME: (ptr noalias nocapture nofree readonly [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[N:%.*]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[P:%.*]], i64 [[Q:%.*]]) {
; CGSCC-NEXT: entry:
-; CGSCC-NEXT: [[Q_ADDR:%.*]] = alloca i64, align 8
+; CGSCC-NEXT: [[Q_ADDR1:%.*]] = alloca i8, i32 8, align 8
; CGSCC-NEXT: [[DOTOMP_LB:%.*]] = alloca i32, align 4
; CGSCC-NEXT: [[DOTOMP_UB:%.*]] = alloca i32, align 4
; CGSCC-NEXT: [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
; CGSCC-NEXT: [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-; CGSCC-NEXT: store i64 4617315517961601024, ptr [[Q_ADDR]], align 8
+; CGSCC-NEXT: store i64 4617315517961601024, ptr [[Q_ADDR1]], align 8
; CGSCC-NEXT: [[TMP:%.*]] = load i32, ptr [[N]], align 4
; CGSCC-NEXT: [[SUB3:%.*]] = add nsw i32 [[TMP]], -3
; CGSCC-NEXT: [[CMP:%.*]] = icmp sgt i32 [[TMP]], 2
@@ -160,7 +160,7 @@ define internal void @.omp_outlined.(ptr noalias %.global_tid., ptr noalias %.bo
; CGSCC: omp.inner.for.body:
; CGSCC-NEXT: [[ADD10:%.*]] = add nsw i32 [[DOTOMP_IV_0]], 2
; CGSCC-NEXT: [[TMP10:%.*]] = load float, ptr [[P]], align 4
-; CGSCC-NEXT: [[TMP11:%.*]] = load double, ptr [[Q_ADDR]], align 8
+; CGSCC-NEXT: [[TMP11:%.*]] = load double, ptr [[Q_ADDR1]], align 8
; CGSCC-NEXT: call void @bar(i32 [[ADD10]], float [[TMP10]], double [[TMP11]])
; CGSCC-NEXT: br label [[OMP_BODY_CONTINUE:%.*]]
; CGSCC: omp.body.continue:
diff --git a/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll b/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
index 15a1ed0e62763..10fa92d17eb3b 100644
--- a/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
+++ b/llvm/test/Transforms/Attributor/IPConstantProp/return-argument.ll
@@ -62,8 +62,8 @@ define void @caller(i1 %C) personality ptr @__gxx_personality_v0 {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@caller
; TUNIT-SAME: (i1 [[C:%.*]]) #[[ATTR1:[0-9]+]] personality ptr @__gxx_personality_v0 {
-; TUNIT-NEXT: [[Q:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: [[W:%.*]] = call align 4 ptr @incdec(i1 noundef [[C]], ptr noalias nofree noundef nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[Q]]) #[[ATTR2:[0-9]+]]
+; TUNIT-NEXT: [[Q1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: [[W:%.*]] = call align 4 ptr @incdec(i1 noundef [[C]], ptr noalias nofree noundef nonnull writeonly align 4 dereferenceable(4) "no-capture-maybe-returned" [[Q1]]) #[[ATTR2:[0-9]+]]
; TUNIT-NEXT: br label [[OK:%.*]]
; TUNIT: OK:
; TUNIT-NEXT: br label [[RET:%.*]]
diff --git a/llvm/test/Transforms/Attributor/align.ll b/llvm/test/Transforms/Attributor/align.ll
index 9880e53fd43a5..4480dd6d71427 100644
--- a/llvm/test/Transforms/Attributor/align.ll
+++ b/llvm/test/Transforms/Attributor/align.ll
@@ -1039,35 +1039,35 @@ define internal ptr @aligned_8_return(ptr %a, i1 %c1, i1 %c2) norecurse {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@aligned_8_return
; TUNIT-SAME: (ptr noalias nofree readnone align 16 "no-capture-maybe-returned" [[A:%.*]], i1 noundef [[C1:%.*]], i1 [[C2:%.*]]) #[[ATTR10]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca ptr, align 8
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, i32 8, align 8
; TUNIT-NEXT: br i1 [[C1]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
; TUNIT-NEXT: [[GEP:%.*]] = getelementptr i8, ptr @G, i32 8
; TUNIT-NEXT: [[SEL:%.*]] = select i1 [[C2]], ptr [[A]], ptr [[GEP]]
-; TUNIT-NEXT: store ptr [[SEL]], ptr [[STACK]], align 8
+; TUNIT-NEXT: store ptr [[SEL]], ptr [[STACK1]], align 8
; TUNIT-NEXT: br label [[END:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store ptr @G, ptr [[STACK]], align 8
+; TUNIT-NEXT: store ptr @G, ptr [[STACK1]], align 8
; TUNIT-NEXT: br label [[END]]
; TUNIT: end:
-; TUNIT-NEXT: [[L:%.*]] = load ptr, ptr [[STACK]], align 8
+; TUNIT-NEXT: [[L:%.*]] = load ptr, ptr [[STACK1]], align 8
; TUNIT-NEXT: ret ptr [[L]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define {{[^@]+}}@aligned_8_return
; CGSCC-SAME: (ptr noalias nofree readnone align 16 "no-capture-maybe-returned" [[A:%.*]], i1 noundef [[C1:%.*]], i1 [[C2:%.*]]) #[[ATTR11]] {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca ptr, align 8
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, i32 8, align 8
; CGSCC-NEXT: br i1 [[C1]], label [[T:%.*]], label [[F:%.*]]
; CGSCC: t:
; CGSCC-NEXT: [[GEP:%.*]] = getelementptr i8, ptr @G, i32 8
; CGSCC-NEXT: [[SEL:%.*]] = select i1 [[C2]], ptr [[A]], ptr [[GEP]]
-; CGSCC-NEXT: store ptr [[SEL]], ptr [[STACK]], align 8
+; CGSCC-NEXT: store ptr [[SEL]], ptr [[STACK1]], align 8
; CGSCC-NEXT: br label [[END:%.*]]
; CGSCC: f:
-; CGSCC-NEXT: store ptr @G, ptr [[STACK]], align 8
+; CGSCC-NEXT: store ptr @G, ptr [[STACK1]], align 8
; CGSCC-NEXT: br label [[END]]
; CGSCC: end:
-; CGSCC-NEXT: [[L:%.*]] = load ptr, ptr [[STACK]], align 8
+; CGSCC-NEXT: [[L:%.*]] = load ptr, ptr [[STACK1]], align 8
; CGSCC-NEXT: ret ptr [[L]]
;
%stack = alloca ptr
diff --git a/llvm/test/Transforms/Attributor/allocator.ll b/llvm/test/Transforms/Attributor/allocator.ll
index f2d9ecd1d8fa4..860f4a63e6754 100644
--- a/llvm/test/Transforms/Attributor/allocator.ll
+++ b/llvm/test/Transforms/Attributor/allocator.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals --version 2
-; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
-; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,CGSCC
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -debug-only=attributor -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal -debug-only=attributor -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,CGSCC
%struct.Foo = type { i32, i32, i8 }
@@ -13,14 +13,14 @@ define dso_local void @positive_alloca_1(i32 noundef %val) #0 {
; CHECK-LABEL: define dso_local void @positive_alloca_1
; CHECK-SAME: (i32 noundef [[VAL:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VAL_ADDR1:%.*]] = alloca i8, i32 4, align 4
-; CHECK-NEXT: [[F2:%.*]] = alloca i8, i32 4, align 4
-; CHECK-NEXT: store i32 [[VAL]], ptr [[VAL_ADDR1]], align 4
-; CHECK-NEXT: store i32 10, ptr [[F2]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[F2]], align 4
+; CHECK-NEXT: [[VAL_ADDR:%.*]] = alloca i64, align 4
+; CHECK-NEXT: [[F:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
+; CHECK-NEXT: store i32 [[VAL]], ptr [[VAL_ADDR]], align 4
+; CHECK-NEXT: store i32 10, ptr [[F]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[F]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], 1
-; CHECK-NEXT: store i32 [[ADD]], ptr [[F2]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
+; CHECK-NEXT: store i32 [[ADD]], ptr [[F]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[F]], align 4
; CHECK-NEXT: [[ADD3:%.*]] = add nsw i32 [[TMP1]], [[VAL]]
; CHECK-NEXT: [[CALL:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 noundef [[ADD3]])
; CHECK-NEXT: ret void
@@ -164,37 +164,45 @@ entry:
;TODO: The allocation can be reduced here.
;However, the offsets (load/store etc.) Need to be changed.
; Function Attrs: noinline nounwind uwtable
-define dso_local { i64, ptr } @positive_test_not_a_single_start_offset(i32 noundef %val) #0 {
-; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
-; CHECK-LABEL: define dso_local { i64, ptr } @positive_test_not_a_single_start_offset
-; CHECK-SAME: (i32 noundef [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+define dso_local void @positive_test_not_a_single_start_offset(i32 noundef %val) #0 {
+; CHECK-LABEL: define dso_local void @positive_test_not_a_single_start_offset
+; CHECK-SAME: (i32 noundef [[VAL:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[RETVAL:%.*]] = alloca [[STRUCT_FOO:%.*]], align 8
; CHECK-NEXT: [[VAL_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[F:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
; CHECK-NEXT: store i32 [[VAL]], ptr [[VAL_ADDR]], align 4
-; CHECK-NEXT: store i32 2, ptr [[RETVAL]], align 8
-; CHECK-NEXT: [[FIELD3:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr [[RETVAL]], i32 0, i32 2
-; CHECK-NEXT: store ptr [[VAL_ADDR]], ptr [[FIELD3]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = load { i64, ptr }, ptr [[RETVAL]], align 8
-; CHECK-NEXT: ret { i64, ptr } [[TMP0]]
+; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 2, [[VAL]]
+; CHECK-NEXT: store i32 [[MUL]], ptr [[F]], align 4
+; CHECK-NEXT: [[CALL:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 noundef [[MUL]])
+; CHECK-NEXT: [[C:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr [[F]], i32 0, i32 2
+; CHECK-NEXT: [[CALL3:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 noundef 67)
+; CHECK-NEXT: ret void
;
entry:
- %retval = alloca %struct.Foo, align 8
%val.addr = alloca i32, align 4
+ %f = alloca %struct.Foo, align 4
store i32 %val, ptr %val.addr, align 4
- %field1 = getelementptr inbounds %struct.Foo, ptr %retval, i32 0, i32 0
- store i32 2, ptr %field1, align 8
- %field3 = getelementptr inbounds %struct.Foo, ptr %retval, i32 0, i32 2
- store ptr %val.addr, ptr %field3, align 8
- %0 = load { i64, ptr }, ptr %retval, align 8
- ret { i64, ptr } %0
+ %0 = load i32, ptr %val.addr, align 4
+ %mul = mul nsw i32 2, %0
+ %a = getelementptr inbounds %struct.Foo, ptr %f, i32 0, i32 0
+ store i32 %mul, ptr %a, align 4
+ %a1 = getelementptr inbounds %struct.Foo, ptr %f, i32 0, i32 0
+ %1 = load i32, ptr %a1, align 4
+ %call = call i32 (ptr, ...) @printf(ptr noundef @.str, i32 noundef %1)
+ %c = getelementptr inbounds %struct.Foo, ptr %f, i32 0, i32 2
+ store i8 67, ptr %c, align 4
+ %c2 = getelementptr inbounds %struct.Foo, ptr %f, i32 0, i32 2
+ %2 = load i8, ptr %c2, align 4
+ %conv = sext i8 %2 to i32
+ %call3 = call i32 (ptr, ...) @printf(ptr noundef @.str, i32 noundef %conv)
+ ret void
}
; Function Attrs: noinline nounwind uwtable
define dso_local void @positive_test_reduce_array_allocation_1() {
; CHECK-LABEL: define dso_local void @positive_test_reduce_array_allocation_1() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[ARRAY1:%.*]] = alloca i8, i32 4, align 8
+; CHECK-NEXT: [[ARRAY1:%.*]] = alloca ptr, i32 10, align 8
; CHECK-NEXT: store i32 0, ptr [[ARRAY1]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAY1]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 2
@@ -275,37 +283,37 @@ entry:
define dso_local void @positive_test_reduce_array_allocation_2() #0 {
; CHECK-LABEL: define dso_local void @positive_test_reduce_array_allocation_2() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[ARRAY:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[ARRAY1:%.*]] = alloca ptr, align 8
+; CHECK-NEXT: [[I2:%.*]] = alloca i32, align 4
; CHECK-NEXT: [[CALL:%.*]] = call noalias ptr @malloc(i64 noundef 40000)
-; CHECK-NEXT: store ptr [[CALL]], ptr [[ARRAY]], align 8
-; CHECK-NEXT: store i32 0, ptr [[I]], align 4
+; CHECK-NEXT: store ptr [[CALL]], ptr [[ARRAY1]], align 8
+; CHECK-NEXT: store i32 0, ptr [[I2]], align 4
; CHECK-NEXT: br label [[FOR_COND:%.*]]
; CHECK: for.cond:
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10000
; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
; CHECK: for.body:
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I2]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[IDXPROM:%.*]] = sext i32 [[TMP2]] to i64
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[CALL]], i64 [[IDXPROM]]
; CHECK-NEXT: store i32 [[TMP1]], ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: br label [[FOR_INC:%.*]]
; CHECK: for.inc:
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP3]], 2
-; CHECK-NEXT: store i32 [[ADD]], ptr [[I]], align 4
+; CHECK-NEXT: store i32 [[ADD]], ptr [[I2]], align 4
; CHECK-NEXT: br label [[FOR_COND]]
; CHECK: for.end:
-; CHECK-NEXT: store i32 0, ptr [[I]], align 4
+; CHECK-NEXT: store i32 0, ptr [[I2]], align 4
; CHECK-NEXT: br label [[FOR_COND1:%.*]]
; CHECK: for.cond1:
-; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[TMP4]], 10000
; CHECK-NEXT: br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END9:%.*]]
; CHECK: for.body3:
-; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[IDXPROM4:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[CALL]], i64 [[IDXPROM4]]
; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
@@ -313,28 +321,28 @@ define dso_local void @positive_test_reduce_array_allocation_2() #0 {
; CHECK-NEXT: store i32 [[ADD6]], ptr [[ARRAYIDX5]], align 4
; CHECK-NEXT: br label [[FOR_INC7:%.*]]
; CHECK: for.inc7:
-; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[ADD8:%.*]] = add nsw i32 [[TMP7]], 2
-; CHECK-NEXT: store i32 [[ADD8]], ptr [[I]], align 4
+; CHECK-NEXT: store i32 [[ADD8]], ptr [[I2]], align 4
; CHECK-NEXT: br label [[FOR_COND1]]
; CHECK: for.end9:
-; CHECK-NEXT: store i32 0, ptr [[I]], align 4
+; CHECK-NEXT: store i32 0, ptr [[I2]], align 4
; CHECK-NEXT: br label [[FOR_COND10:%.*]]
; CHECK: for.cond10:
-; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[CMP11:%.*]] = icmp slt i32 [[TMP8]], 10000
; CHECK-NEXT: br i1 [[CMP11]], label [[FOR_BODY12:%.*]], label [[FOR_END18:%.*]]
; CHECK: for.body12:
-; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[IDXPROM13:%.*]] = sext i32 [[TMP9]] to i64
; CHECK-NEXT: [[ARRAYIDX14:%.*]] = getelementptr inbounds i32, ptr [[CALL]], i64 [[IDXPROM13]]
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX14]], align 4
; CHECK-NEXT: [[CALL15:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 noundef [[TMP10]])
; CHECK-NEXT: br label [[FOR_INC16:%.*]]
; CHECK: for.inc16:
-; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[ADD17:%.*]] = add nsw i32 [[TMP11]], 2
-; CHECK-NEXT: store i32 [[ADD17]], ptr [[I]], align 4
+; CHECK-NEXT: store i32 [[ADD17]], ptr [[I2]], align 4
; CHECK-NEXT: br label [[FOR_COND10]]
; CHECK: for.end18:
; CHECK-NEXT: ret void
@@ -425,21 +433,20 @@ define dso_local void @pthread_test(){
; TUNIT-LABEL: define dso_local void @pthread_test() {
; TUNIT-NEXT: [[ARG1:%.*]] = alloca i8, align 8
; TUNIT-NEXT: [[THREAD:%.*]] = alloca i64, align 8
-; TUNIT-NEXT: [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_remain_same, ptr noundef nonnull align 8 dereferenceable(1) [[ARG1]])
-; TUNIT-NEXT: [[F1:%.*]] = alloca i8, i32 4, align 4
-; TUNIT-NEXT: [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(12) undef)
+; TUNIT-NEXT: [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_remain_same, ptr noundef nonnull align 8 dereferenceable(1) [[ARG1]])
+; TUNIT-NEXT: [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(12) undef)
; TUNIT-NEXT: [[F2:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
-; TUNIT-NEXT: [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_check_captured_pointer, ptr noundef nonnull align 4 dereferenceable(12) [[F2]])
+; TUNIT-NEXT: [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_check_captured_pointer, ptr noundef nonnull align 4 dereferenceable(12) [[F2]])
; TUNIT-NEXT: ret void
;
; CGSCC-LABEL: define dso_local void @pthread_test() {
; CGSCC-NEXT: [[ARG1:%.*]] = alloca i8, align 8
; CGSCC-NEXT: [[THREAD:%.*]] = alloca i64, align 8
-; CGSCC-NEXT: [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_remain_same, ptr noundef nonnull align 8 dereferenceable(1) [[ARG1]])
+; CGSCC-NEXT: [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_remain_same, ptr noundef nonnull align 8 dereferenceable(1) [[ARG1]])
; CGSCC-NEXT: [[F:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
-; CGSCC-NEXT: [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(12) [[F]])
+; CGSCC-NEXT: [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr nofree noundef nonnull readonly align 4 dereferenceable(12) [[F]])
; CGSCC-NEXT: [[F2:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CGSCC-NEXT: [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_check_captured_pointer, ptr noundef nonnull align 4 dereferenceable(12) [[F2]])
+; CGSCC-NEXT: [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_check_captured_pointer, ptr noundef nonnull align 4 dereferenceable(12) [[F2]])
; CGSCC-NEXT: ret void
;
%arg1 = alloca i8, align 8
@@ -499,6 +506,58 @@ entry:
ret void
}
+define dso_local void @alloca_array_multi_offset(){
+; CHECK: Function Attrs: nofree norecurse nosync nounwind memory(none)
+; CHECK-LABEL: define dso_local void @alloca_array_multi_offset
+; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: store i32 0, ptr [[I]], align 4
+; CHECK-NEXT: br label [[FOR_COND:%.*]]
+; CHECK: for.cond:
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
+; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: br label [[FOR_INC:%.*]]
+; CHECK: for.inc:
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], 2
+; CHECK-NEXT: store i32 [[ADD]], ptr [[I]], align 4
+; CHECK-NEXT: br label [[FOR_COND]]
+; CHECK: for.end:
+; CHECK-NEXT: ret void
+;
+entry:
+ %arr = alloca i8, i32 10, align 4
+ %i = alloca i32, align 4
+ store i32 0, ptr %i, align 4
+ br label %for.cond
+
+for.cond:
+ %0 = load i32, ptr %i, align 4
+ %cmp = icmp slt i32 %0, 10
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+ %1 = load i32, ptr %i, align 4
+ %2 = load ptr, ptr %arr, align 8
+ %3 = load i32, ptr %i, align 4
+ %arrayidx = getelementptr inbounds i32, ptr %2, i32 %3
+ store i32 %1, ptr %arrayidx, align 4
+ br label %for.inc
+
+for.inc:
+ %4 = load i32, ptr %i, align 4
+ %add = add nsw i32 %4, 2
+ store i32 %add, ptr %i, align 4
+ br label %for.cond
+
+for.end:
+ ret void
+
+}
+
declare external void @external_call(ptr)
diff --git a/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll b/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll
index 5bb795911ce40..b145c3b720186 100644
--- a/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll
+++ b/llvm/test/Transforms/Attributor/call-simplify-pointer-info.ll
@@ -36,8 +36,12 @@ define i8 @call_simplifiable_1() {
; TUNIT-LABEL: define {{[^@]+}}@call_simplifiable_1
; TUNIT-SAME: () #[[ATTR0:[0-9]+]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[BYTES:%.*]] = alloca [1024 x i8], align 16
-; TUNIT-NEXT: [[I0:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 2
+; TUNIT-NEXT: [[BYTES1:%.*]] = alloca i8, align 16
+; TUNIT-NEXT: [[I0:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 2
+; TUNIT-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[I0]] to i32
+; TUNIT-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 2
+; TUNIT-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP1]] to ptr
+; TUNIT-NEXT: store i8 2, ptr [[TMP2]], align 1
; TUNIT-NEXT: ret i8 2
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -93,9 +97,17 @@ define i8 @call_simplifiable_2() {
; TUNIT-LABEL: define {{[^@]+}}@call_simplifiable_2
; TUNIT-SAME: () #[[ATTR0]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[BYTES:%.*]] = alloca [1024 x i8], align 16
-; TUNIT-NEXT: [[I0:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 2
-; TUNIT-NEXT: [[I1:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 3
+; TUNIT-NEXT: [[BYTES1:%.*]] = alloca i8, i32 2, align 16
+; TUNIT-NEXT: [[I0:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 2
+; TUNIT-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[I0]] to i32
+; TUNIT-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 2
+; TUNIT-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP1]] to ptr
+; TUNIT-NEXT: store i8 2, ptr [[TMP2]], align 1
+; TUNIT-NEXT: [[I1:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 3
+; TUNIT-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[I1]] to i32
+; TUNIT-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 2
+; TUNIT-NEXT: [[TMP5:%.*]] = inttoptr i32 [[TMP4]] to ptr
+; TUNIT-NEXT: store i8 3, ptr [[TMP5]], align 1
; TUNIT-NEXT: ret i8 4
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -125,8 +137,12 @@ define i8 @call_simplifiable_3() {
; TUNIT-LABEL: define {{[^@]+}}@call_simplifiable_3
; TUNIT-SAME: () #[[ATTR0]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[BYTES:%.*]] = alloca [1024 x i8], align 16
-; TUNIT-NEXT: [[I2:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 2
+; TUNIT-NEXT: [[BYTES1:%.*]] = alloca i8, align 16
+; TUNIT-NEXT: [[I2:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 2
+; TUNIT-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[I2]] to i32
+; TUNIT-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 2
+; TUNIT-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP1]] to ptr
+; TUNIT-NEXT: store i8 2, ptr [[TMP2]], align 1
; TUNIT-NEXT: ret i8 2
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -198,12 +214,24 @@ define i8 @call_partially_simplifiable_1() {
; TUNIT-LABEL: define {{[^@]+}}@call_partially_simplifiable_1
; TUNIT-SAME: () #[[ATTR0]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[BYTES:%.*]] = alloca [1024 x i8], align 16
-; TUNIT-NEXT: [[I2:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 2
+; TUNIT-NEXT: [[BYTES1:%.*]] = alloca i8, i32 3, align 16
+; TUNIT-NEXT: [[I2:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 2
+; TUNIT-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[I2]] to i32
+; TUNIT-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 2
+; TUNIT-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP1]] to ptr
+; TUNIT-NEXT: store i8 2, ptr [[TMP2]], align 1
; TUNIT-NEXT: store i8 2, ptr [[I2]], align 2
-; TUNIT-NEXT: [[I3:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 3
+; TUNIT-NEXT: [[I3:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 3
+; TUNIT-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[I3]] to i32
+; TUNIT-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; TUNIT-NEXT: [[TMP5:%.*]] = inttoptr i32 [[TMP4]] to ptr
+; TUNIT-NEXT: store i8 3, ptr [[TMP5]], align 1
; TUNIT-NEXT: store i8 3, ptr [[I3]], align 1
-; TUNIT-NEXT: [[I4:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 4
+; TUNIT-NEXT: [[I4:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 4
+; TUNIT-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[I4]] to i32
+; TUNIT-NEXT: [[TMP7:%.*]] = sub i32 [[TMP6]], 3
+; TUNIT-NEXT: [[TMP8:%.*]] = inttoptr i32 [[TMP7]] to ptr
+; TUNIT-NEXT: store i8 4, ptr [[TMP8]], align 1
; TUNIT-NEXT: [[R:%.*]] = call i8 @sum_two_different_loads(ptr nocapture nofree noundef nonnull readonly align 2 dereferenceable(1022) [[I2]], ptr nocapture nofree noundef nonnull readonly dereferenceable(1021) [[I3]]) #[[ATTR3]]
; TUNIT-NEXT: ret i8 [[R]]
;
@@ -239,13 +267,25 @@ define i8 @call_partially_simplifiable_2(i1 %cond) {
; TUNIT-LABEL: define {{[^@]+}}@call_partially_simplifiable_2
; TUNIT-SAME: (i1 [[COND:%.*]]) #[[ATTR2:[0-9]+]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[BYTES:%.*]] = alloca [1024 x i8], align 16
-; TUNIT-NEXT: [[I51:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 51
-; TUNIT-NEXT: [[I52:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 52
+; TUNIT-NEXT: [[BYTES1:%.*]] = alloca i8, i32 4, align 16
+; TUNIT-NEXT: [[I51:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 51
+; TUNIT-NEXT: [[I52:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 52
+; TUNIT-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[I52]] to i32
+; TUNIT-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], 52
+; TUNIT-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP1]] to ptr
+; TUNIT-NEXT: store i8 2, ptr [[TMP2]], align 1
; TUNIT-NEXT: store i8 2, ptr [[I52]], align 4
-; TUNIT-NEXT: [[I53:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 53
+; TUNIT-NEXT: [[I53:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 53
+; TUNIT-NEXT: [[TMP3:%.*]] = ptrtoint ptr [[I53]] to i32
+; TUNIT-NEXT: [[TMP4:%.*]] = sub i32 [[TMP3]], 50
+; TUNIT-NEXT: [[TMP5:%.*]] = inttoptr i32 [[TMP4]] to ptr
+; TUNIT-NEXT: store i8 3, ptr [[TMP5]], align 1
; TUNIT-NEXT: store i8 3, ptr [[I53]], align 1
-; TUNIT-NEXT: [[I54:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES]], i64 0, i64 54
+; TUNIT-NEXT: [[I54:%.*]] = getelementptr inbounds [1024 x i8], ptr [[BYTES1]], i64 0, i64 54
+; TUNIT-NEXT: [[TMP6:%.*]] = ptrtoint ptr [[I54]] to i32
+; TUNIT-NEXT: [[TMP7:%.*]] = sub i32 [[TMP6]], 53
+; TUNIT-NEXT: [[TMP8:%.*]] = inttoptr i32 [[TMP7]] to ptr
+; TUNIT-NEXT: store i8 4, ptr [[TMP8]], align 1
; TUNIT-NEXT: [[SEL:%.*]] = select i1 [[COND]], ptr [[I51]], ptr [[I52]]
; TUNIT-NEXT: [[R:%.*]] = call i8 @sum_two_different_loads(ptr nocapture nofree nonnull readonly dereferenceable(972) [[SEL]], ptr nocapture nofree noundef nonnull readonly dereferenceable(971) [[I53]]) #[[ATTR3]]
; TUNIT-NEXT: ret i8 [[R]]
diff --git a/llvm/test/Transforms/Attributor/callbacks.ll b/llvm/test/Transforms/Attributor/callbacks.ll
index dd5cbbc9e271e..d5f7d09b2c8d8 100644
--- a/llvm/test/Transforms/Attributor/callbacks.ll
+++ b/llvm/test/Transforms/Attributor/callbacks.ll
@@ -18,11 +18,11 @@ define void @t0_caller(ptr %a) {
; TUNIT-SAME: (ptr align 256 [[A:%.*]]) {
; TUNIT-NEXT: entry:
; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 32
-; TUNIT-NEXT: [[C:%.*]] = alloca ptr, align 64
+; TUNIT-NEXT: [[C1:%.*]] = alloca i8, i32 8, align 64
; TUNIT-NEXT: [[PTR:%.*]] = alloca i32, align 128
; TUNIT-NEXT: store i32 42, ptr [[B]], align 32
-; TUNIT-NEXT: store ptr [[B]], ptr [[C]], align 64
-; TUNIT-NEXT: call void (ptr, ptr, ptr, ...) @t0_callback_broker(ptr noundef align 4294967296 null, ptr noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr noundef nonnull @t0_callback_callee, ptr align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; TUNIT-NEXT: store ptr [[B]], ptr [[C1]], align 64
+; TUNIT-NEXT: call void (ptr, ptr, ptr, ...) @t0_callback_broker(ptr noundef align 4294967296 null, ptr noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr noundef nonnull @t0_callback_callee, ptr align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C1]])
; TUNIT-NEXT: ret void
;
; CGSCC-LABEL: define {{[^@]+}}@t0_caller
@@ -90,12 +90,16 @@ define void @t1_caller(ptr noalias %a) {
; TUNIT-LABEL: define {{[^@]+}}@t1_caller
; TUNIT-SAME: (ptr noalias nocapture align 256 [[A:%.*]]) {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 32
-; TUNIT-NEXT: [[C:%.*]] = alloca ptr, align 64
-; TUNIT-NEXT: [[PTR:%.*]] = alloca i32, align 128
-; TUNIT-NEXT: store i32 42, ptr [[B]], align 32
-; TUNIT-NEXT: store ptr [[B]], ptr [[C]], align 64
-; TUNIT-NEXT: call void (ptr, ptr, ptr, ...) @t1_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t1_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; TUNIT-NEXT: [[B1:%.*]] = alloca i8, i32 -2147483645, align 32
+; TUNIT-NEXT: [[C2:%.*]] = alloca i8, i32 8, align 64
+; TUNIT-NEXT: [[PTR3:%.*]] = alloca i8, i32 2147483647, align 128
+; TUNIT-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[B1]] to i32
+; TUNIT-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], -2147483647
+; TUNIT-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP1]] to ptr
+; TUNIT-NEXT: store i32 42, ptr [[TMP2]], align 4
+; TUNIT-NEXT: store i32 42, ptr [[B1]], align 32
+; TUNIT-NEXT: store ptr [[B1]], ptr [[C2]], align 64
+; TUNIT-NEXT: call void (ptr, ptr, ptr, ...) @t1_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR3]], ptr nocapture noundef nonnull @t1_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C2]])
; TUNIT-NEXT: ret void
;
; CGSCC-LABEL: define {{[^@]+}}@t1_caller
@@ -103,10 +107,10 @@ define void @t1_caller(ptr noalias %a) {
; CGSCC-NEXT: entry:
; CGSCC-NEXT: [[B:%.*]] = alloca i32, align 32
; CGSCC-NEXT: [[C:%.*]] = alloca ptr, align 64
-; CGSCC-NEXT: [[PTR:%.*]] = alloca i32, align 128
+; CGSCC-NEXT: [[PTR1:%.*]] = alloca i8, i32 2147483647, align 128
; CGSCC-NEXT: store i32 42, ptr [[B]], align 32
; CGSCC-NEXT: store ptr [[B]], ptr [[C]], align 64
-; CGSCC-NEXT: call void (ptr, ptr, ptr, ...) @t1_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t1_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; CGSCC-NEXT: call void (ptr, ptr, ptr, ...) @t1_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR1]], ptr nocapture noundef nonnull @t1_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
; CGSCC-NEXT: ret void
;
entry:
@@ -163,12 +167,16 @@ define void @t2_caller(ptr noalias %a) {
; TUNIT-LABEL: define {{[^@]+}}@t2_caller
; TUNIT-SAME: (ptr noalias nocapture align 256 [[A:%.*]]) {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 32
-; TUNIT-NEXT: [[C:%.*]] = alloca ptr, align 64
-; TUNIT-NEXT: [[PTR:%.*]] = alloca i32, align 128
-; TUNIT-NEXT: store i32 42, ptr [[B]], align 32
-; TUNIT-NEXT: store ptr [[B]], ptr [[C]], align 64
-; TUNIT-NEXT: call void (ptr, ptr, ptr, ...) @t2_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t2_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; TUNIT-NEXT: [[B1:%.*]] = alloca i8, i32 -2147483645, align 32
+; TUNIT-NEXT: [[C2:%.*]] = alloca i8, i32 8, align 64
+; TUNIT-NEXT: [[PTR3:%.*]] = alloca i8, i32 2147483647, align 128
+; TUNIT-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[B1]] to i32
+; TUNIT-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], -2147483647
+; TUNIT-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP1]] to ptr
+; TUNIT-NEXT: store i32 42, ptr [[TMP2]], align 4
+; TUNIT-NEXT: store i32 42, ptr [[B1]], align 32
+; TUNIT-NEXT: store ptr [[B1]], ptr [[C2]], align 64
+; TUNIT-NEXT: call void (ptr, ptr, ptr, ...) @t2_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR3]], ptr nocapture noundef nonnull @t2_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C2]])
; TUNIT-NEXT: ret void
;
; CGSCC-LABEL: define {{[^@]+}}@t2_caller
@@ -176,10 +184,10 @@ define void @t2_caller(ptr noalias %a) {
; CGSCC-NEXT: entry:
; CGSCC-NEXT: [[B:%.*]] = alloca i32, align 32
; CGSCC-NEXT: [[C:%.*]] = alloca ptr, align 64
-; CGSCC-NEXT: [[PTR:%.*]] = alloca i32, align 128
+; CGSCC-NEXT: [[PTR1:%.*]] = alloca i8, i32 2147483647, align 128
; CGSCC-NEXT: store i32 42, ptr [[B]], align 32
; CGSCC-NEXT: store ptr [[B]], ptr [[C]], align 64
-; CGSCC-NEXT: call void (ptr, ptr, ptr, ...) @t2_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t2_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; CGSCC-NEXT: call void (ptr, ptr, ptr, ...) @t2_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR1]], ptr nocapture noundef nonnull @t2_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
; CGSCC-NEXT: ret void
;
entry:
@@ -236,13 +244,17 @@ define void @t3_caller(ptr noalias %a) {
; TUNIT-LABEL: define {{[^@]+}}@t3_caller
; TUNIT-SAME: (ptr noalias nocapture align 256 [[A:%.*]]) {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 32
-; TUNIT-NEXT: [[C:%.*]] = alloca ptr, align 64
-; TUNIT-NEXT: [[PTR:%.*]] = alloca i32, align 128
-; TUNIT-NEXT: store i32 42, ptr [[B]], align 32
-; TUNIT-NEXT: store ptr [[B]], ptr [[C]], align 64
-; TUNIT-NEXT: call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
-; TUNIT-NEXT: call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; TUNIT-NEXT: [[B1:%.*]] = alloca i8, i32 -2147483645, align 32
+; TUNIT-NEXT: [[C2:%.*]] = alloca i8, i32 8, align 64
+; TUNIT-NEXT: [[PTR3:%.*]] = alloca i8, i32 2147483647, align 128
+; TUNIT-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[B1]] to i32
+; TUNIT-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], -2147483647
+; TUNIT-NEXT: [[TMP2:%.*]] = inttoptr i32 [[TMP1]] to ptr
+; TUNIT-NEXT: store i32 42, ptr [[TMP2]], align 4
+; TUNIT-NEXT: store i32 42, ptr [[B1]], align 32
+; TUNIT-NEXT: store ptr [[B1]], ptr [[C2]], align 64
+; TUNIT-NEXT: call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR3]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C2]])
+; TUNIT-NEXT: call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR3]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 undef, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C2]])
; TUNIT-NEXT: ret void
;
; CGSCC-LABEL: define {{[^@]+}}@t3_caller
@@ -250,11 +262,11 @@ define void @t3_caller(ptr noalias %a) {
; CGSCC-NEXT: entry:
; CGSCC-NEXT: [[B:%.*]] = alloca i32, align 32
; CGSCC-NEXT: [[C:%.*]] = alloca ptr, align 64
-; CGSCC-NEXT: [[PTR:%.*]] = alloca i32, align 128
+; CGSCC-NEXT: [[PTR1:%.*]] = alloca i8, i32 2147483647, align 128
; CGSCC-NEXT: store i32 42, ptr [[B]], align 32
; CGSCC-NEXT: store ptr [[B]], ptr [[C]], align 64
-; CGSCC-NEXT: call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
-; CGSCC-NEXT: call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; CGSCC-NEXT: call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR1]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
+; CGSCC-NEXT: call void (ptr, ptr, ptr, ...) @t3_callback_broker(ptr noundef align 4294967296 null, ptr noalias nocapture noundef nonnull align 128 dereferenceable(4) [[PTR1]], ptr nocapture noundef nonnull @t3_callback_callee, ptr nocapture align 256 [[A]], i64 noundef 99, ptr noalias nocapture nofree noundef nonnull readonly align 64 dereferenceable(8) [[C]])
; CGSCC-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack.ll b/llvm/test/Transforms/Attributor/heap_to_stack.ll
index 33ac066e43d09..8ea7923f0dc77 100644
--- a/llvm/test/Transforms/Attributor/heap_to_stack.ll
+++ b/llvm/test/Transforms/Attributor/heap_to_stack.ll
@@ -501,15 +501,15 @@ define i32 @malloc_in_loop(i32 %arg) {
; CHECK-LABEL: define {{[^@]+}}@malloc_in_loop
; CHECK-SAME: (i32 [[ARG:%.*]]) {
; CHECK-NEXT: bb:
-; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[I2:%.*]] = alloca i8, i32 4, align 4
; CHECK-NEXT: [[I1:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: [[I11:%.*]] = alloca i8, i32 0, align 8
-; CHECK-NEXT: store i32 [[ARG]], ptr [[I]], align 4
+; CHECK-NEXT: [[I13:%.*]] = alloca i8, i32 0, align 8
+; CHECK-NEXT: store i32 [[ARG]], ptr [[I2]], align 4
; CHECK-NEXT: br label [[BB2:%.*]]
; CHECK: bb2:
-; CHECK-NEXT: [[I3:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[I3:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[I4:%.*]] = add nsw i32 [[I3]], -1
-; CHECK-NEXT: store i32 [[I4]], ptr [[I]], align 4
+; CHECK-NEXT: store i32 [[I4]], ptr [[I2]], align 4
; CHECK-NEXT: [[I5:%.*]] = icmp sgt i32 [[I4]], 0
; CHECK-NEXT: br i1 [[I5]], label [[BB6:%.*]], label [[BB9:%.*]]
; CHECK: bb6:
diff --git a/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll b/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll
index 2a5b3e94291a2..57f36a05e30b8 100644
--- a/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll
+++ b/llvm/test/Transforms/Attributor/heap_to_stack_gpu.ll
@@ -451,15 +451,15 @@ define i32 @malloc_in_loop(i32 %arg) {
; CHECK-LABEL: define {{[^@]+}}@malloc_in_loop
; CHECK-SAME: (i32 [[ARG:%.*]]) {
; CHECK-NEXT: bb:
-; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[I2:%.*]] = alloca i8, i32 4, align 4
; CHECK-NEXT: [[I1:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: [[I11:%.*]] = alloca i8, i32 0, align 8
-; CHECK-NEXT: store i32 [[ARG]], ptr [[I]], align 4
+; CHECK-NEXT: [[I13:%.*]] = alloca i8, i32 0, align 8
+; CHECK-NEXT: store i32 [[ARG]], ptr [[I2]], align 4
; CHECK-NEXT: br label [[BB2:%.*]]
; CHECK: bb2:
-; CHECK-NEXT: [[I3:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[I3:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[I4:%.*]] = add nsw i32 [[I3]], -1
-; CHECK-NEXT: store i32 [[I4]], ptr [[I]], align 4
+; CHECK-NEXT: store i32 [[I4]], ptr [[I2]], align 4
; CHECK-NEXT: [[I5:%.*]] = icmp sgt i32 [[I4]], 0
; CHECK-NEXT: br i1 [[I5]], label [[BB6:%.*]], label [[BB9:%.*]]
; CHECK: bb6:
diff --git a/llvm/test/Transforms/Attributor/internal-noalias.ll b/llvm/test/Transforms/Attributor/internal-noalias.ll
index 0dba6853e0c8f..62048de8d0c1d 100644
--- a/llvm/test/Transforms/Attributor/internal-noalias.ll
+++ b/llvm/test/Transforms/Attributor/internal-noalias.ll
@@ -92,10 +92,10 @@ define dso_local i32 @visible_local(ptr %A) #0 {
; TUNIT-LABEL: define {{[^@]+}}@visible_local
; TUNIT-SAME: (ptr nocapture nofree readonly [[A:%.*]]) #[[ATTR1:[0-9]+]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: store i32 5, ptr [[B]], align 4
-; TUNIT-NEXT: [[CALL1:%.*]] = call i32 @noalias_args(ptr nocapture nofree noundef readonly align 4 [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B]]) #[[ATTR4]]
-; TUNIT-NEXT: [[CALL2:%.*]] = call i32 @noalias_args_argmem(ptr nocapture nofree noundef readonly align 4 [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B]]) #[[ATTR4]]
+; TUNIT-NEXT: [[B1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: store i32 5, ptr [[B1]], align 4
+; TUNIT-NEXT: [[CALL1:%.*]] = call i32 @noalias_args(ptr nocapture nofree noundef readonly align 4 [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B1]]) #[[ATTR4]]
+; TUNIT-NEXT: [[CALL2:%.*]] = call i32 @noalias_args_argmem(ptr nocapture nofree noundef readonly align 4 [[A]], ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[B1]]) #[[ATTR4]]
; TUNIT-NEXT: [[ADD:%.*]] = add nsw i32 [[CALL1]], [[CALL2]]
; TUNIT-NEXT: ret i32 [[ADD]]
;
@@ -142,7 +142,7 @@ define i32 @visible_local_2() {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@visible_local_2
; TUNIT-SAME: () #[[ATTR2:[0-9]+]] {
-; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 4
+; TUNIT-NEXT: [[B1:%.*]] = alloca i8, i32 4, align 4
; TUNIT-NEXT: ret i32 10
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -179,8 +179,8 @@ define i32 @visible_local_3() {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@visible_local_3
; TUNIT-SAME: () #[[ATTR2]] {
-; TUNIT-NEXT: [[B:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: [[CALL:%.*]] = call i32 @noalias_args_argmem_rn(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B]]) #[[ATTR5:[0-9]+]]
+; TUNIT-NEXT: [[B1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: [[CALL:%.*]] = call i32 @noalias_args_argmem_rn(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[B1]]) #[[ATTR5:[0-9]+]]
; TUNIT-NEXT: ret i32 5
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
diff --git a/llvm/test/Transforms/Attributor/noalias.ll b/llvm/test/Transforms/Attributor/noalias.ll
index c63d81878f535..759e8f0ba42d3 100644
--- a/llvm/test/Transforms/Attributor/noalias.ll
+++ b/llvm/test/Transforms/Attributor/noalias.ll
@@ -189,21 +189,37 @@ define ptr @test6() nounwind uwtable ssp {
; TUNIT: Function Attrs: nounwind ssp uwtable
; TUNIT-LABEL: define {{[^@]+}}@test6
; TUNIT-SAME: () #[[ATTR3:[0-9]+]] {
-; TUNIT-NEXT: [[X:%.*]] = alloca [2 x i8], align 1
-; TUNIT-NEXT: store i8 97, ptr [[X]], align 1
-; TUNIT-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 0, i64 1
+; TUNIT-NEXT: [[X1:%.*]] = alloca i8, i32 -2147483647, align 1
+; TUNIT-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[X1]] to i32
+; TUNIT-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], -2147483647
+; TUNIT-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr
+; TUNIT-NEXT: store i8 97, ptr [[TMP3]], align 1
+; TUNIT-NEXT: store i8 97, ptr [[X1]], align 1
+; TUNIT-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X1]], i64 0, i64 1
+; TUNIT-NEXT: [[TMP4:%.*]] = ptrtoint ptr [[ARRAYIDX1]] to i32
+; TUNIT-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], -2147483647
+; TUNIT-NEXT: [[TMP6:%.*]] = inttoptr i32 [[TMP5]] to ptr
+; TUNIT-NEXT: store i8 0, ptr [[TMP6]], align 1
; TUNIT-NEXT: store i8 0, ptr [[ARRAYIDX1]], align 1
-; TUNIT-NEXT: [[CALL:%.*]] = call noalias ptr @strdup(ptr noalias nocapture noundef nonnull dereferenceable(2) [[X]]) #[[ATTR2]]
+; TUNIT-NEXT: [[CALL:%.*]] = call noalias ptr @strdup(ptr noalias nocapture noundef nonnull dereferenceable(2) [[X1]]) #[[ATTR2]]
; TUNIT-NEXT: ret ptr [[CALL]]
;
; CGSCC: Function Attrs: nounwind ssp uwtable
; CGSCC-LABEL: define {{[^@]+}}@test6
; CGSCC-SAME: () #[[ATTR4:[0-9]+]] {
-; CGSCC-NEXT: [[X:%.*]] = alloca [2 x i8], align 1
-; CGSCC-NEXT: store i8 97, ptr [[X]], align 1
-; CGSCC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X]], i64 0, i64 1
+; CGSCC-NEXT: [[X1:%.*]] = alloca i8, i32 -2147483647, align 1
+; CGSCC-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[X1]] to i32
+; CGSCC-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], -2147483647
+; CGSCC-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr
+; CGSCC-NEXT: store i8 97, ptr [[TMP3]], align 1
+; CGSCC-NEXT: store i8 97, ptr [[X1]], align 1
+; CGSCC-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [2 x i8], ptr [[X1]], i64 0, i64 1
+; CGSCC-NEXT: [[TMP4:%.*]] = ptrtoint ptr [[ARRAYIDX1]] to i32
+; CGSCC-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], -2147483647
+; CGSCC-NEXT: [[TMP6:%.*]] = inttoptr i32 [[TMP5]] to ptr
+; CGSCC-NEXT: store i8 0, ptr [[TMP6]], align 1
; CGSCC-NEXT: store i8 0, ptr [[ARRAYIDX1]], align 1
-; CGSCC-NEXT: [[CALL:%.*]] = call noalias ptr @strdup(ptr noalias nocapture noundef nonnull dereferenceable(2) [[X]]) #[[ATTR3]]
+; CGSCC-NEXT: [[CALL:%.*]] = call noalias ptr @strdup(ptr noalias nocapture noundef nonnull dereferenceable(2) [[X1]]) #[[ATTR3]]
; CGSCC-NEXT: ret ptr [[CALL]]
;
%x = alloca [2 x i8], align 1
@@ -399,10 +415,10 @@ declare void @use_nocapture(ptr nocapture)
declare void @use(ptr)
define void @test12_1() {
; CHECK-LABEL: define {{[^@]+}}@test12_1() {
-; CHECK-NEXT: [[A:%.*]] = alloca i8, align 4
+; CHECK-NEXT: [[A1:%.*]] = alloca i8, i32 2147483647, align 4
; CHECK-NEXT: [[B:%.*]] = tail call noalias ptr @malloc(i64 noundef 4)
-; CHECK-NEXT: tail call void @use_nocapture(ptr noalias nocapture noundef nonnull align 4 dereferenceable(1) [[A]])
-; CHECK-NEXT: tail call void @use_nocapture(ptr noalias nocapture noundef nonnull align 4 dereferenceable(1) [[A]])
+; CHECK-NEXT: tail call void @use_nocapture(ptr noalias nocapture noundef nonnull align 4 dereferenceable(1) [[A1]])
+; CHECK-NEXT: tail call void @use_nocapture(ptr noalias nocapture noundef nonnull align 4 dereferenceable(1) [[A1]])
; CHECK-NEXT: tail call void @use_nocapture(ptr noalias nocapture [[B]])
; CHECK-NEXT: tail call void @use_nocapture(ptr noalias nocapture [[B]])
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/Attributor/nocapture-2.ll b/llvm/test/Transforms/Attributor/nocapture-2.ll
index 1e5e9da19909f..96fd887f75fac 100644
--- a/llvm/test/Transforms/Attributor/nocapture-2.ll
+++ b/llvm/test/Transforms/Attributor/nocapture-2.ll
@@ -46,21 +46,21 @@ define i32 @is_null_control(ptr %p) #0 {
; CHECK-LABEL: define i32 @is_null_control
; CHECK-SAME: (ptr nofree [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[RETVAL1:%.*]] = alloca i8, i32 4, align 4
; CHECK-NEXT: [[CMP:%.*]] = icmp eq ptr [[P]], null
; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
; CHECK: if.then:
-; CHECK-NEXT: store i32 1, ptr [[RETVAL]], align 4
+; CHECK-NEXT: store i32 1, ptr [[RETVAL1]], align 4
; CHECK-NEXT: br label [[RETURN:%.*]]
; CHECK: if.end:
; CHECK-NEXT: br label [[IF_END3:%.*]]
; CHECK: if.then2:
; CHECK-NEXT: unreachable
; CHECK: if.end3:
-; CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4
+; CHECK-NEXT: store i32 0, ptr [[RETVAL1]], align 4
; CHECK-NEXT: br label [[RETURN]]
; CHECK: return:
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[RETVAL]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[RETVAL1]], align 4
; CHECK-NEXT: ret i32 [[TMP0]]
;
entry:
@@ -737,15 +737,15 @@ define ptr @b64613_a(ptr noundef %p) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define noundef ptr @b64613_a
; TUNIT-SAME: (ptr nofree noundef readnone returned "no-capture-maybe-returned" [[P:%.*]]) #[[ATTR11:[0-9]+]] {
-; TUNIT-NEXT: [[P_ADDR:%.*]] = alloca ptr, align 1
-; TUNIT-NEXT: store ptr [[P]], ptr [[P_ADDR]], align 1
+; TUNIT-NEXT: [[P_ADDR1:%.*]] = alloca i8, i32 8, align 1
+; TUNIT-NEXT: store ptr [[P]], ptr [[P_ADDR1]], align 1
; TUNIT-NEXT: ret ptr [[P]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define noundef ptr @b64613_a
; CGSCC-SAME: (ptr nofree noundef readnone returned "no-capture-maybe-returned" [[P:%.*]]) #[[ATTR12:[0-9]+]] {
-; CGSCC-NEXT: [[P_ADDR:%.*]] = alloca ptr, align 1
-; CGSCC-NEXT: store ptr [[P]], ptr [[P_ADDR]], align 1
+; CGSCC-NEXT: [[P_ADDR1:%.*]] = alloca i8, i32 8, align 1
+; CGSCC-NEXT: store ptr [[P]], ptr [[P_ADDR1]], align 1
; CGSCC-NEXT: ret ptr [[P]]
;
%p.addr = alloca ptr, align 1
@@ -757,19 +757,27 @@ define ptr @b64613_b(ptr noundef %p, i32 %i) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define ptr @b64613_b
; TUNIT-SAME: (ptr nofree noundef [[P:%.*]], i32 [[I:%.*]]) #[[ATTR11]] {
-; TUNIT-NEXT: [[P_ADDR:%.*]] = alloca <2 x ptr>, align 1
-; TUNIT-NEXT: [[G:%.*]] = getelementptr i8, ptr [[P_ADDR]], i32 [[I]]
+; TUNIT-NEXT: [[P_ADDR1:%.*]] = alloca i8, i32 16, align 1
+; TUNIT-NEXT: [[G:%.*]] = getelementptr i8, ptr [[P_ADDR1]], i32 [[I]]
+; TUNIT-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[G]] to i32
+; TUNIT-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], 2147483639
+; TUNIT-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr
+; TUNIT-NEXT: store ptr [[P]], ptr [[TMP3]], align 8
; TUNIT-NEXT: store ptr [[P]], ptr [[G]], align 1
-; TUNIT-NEXT: [[R:%.*]] = load ptr, ptr [[P_ADDR]], align 1
+; TUNIT-NEXT: [[R:%.*]] = load ptr, ptr [[P_ADDR1]], align 1
; TUNIT-NEXT: ret ptr [[R]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define ptr @b64613_b
; CGSCC-SAME: (ptr nofree noundef [[P:%.*]], i32 [[I:%.*]]) #[[ATTR12]] {
-; CGSCC-NEXT: [[P_ADDR:%.*]] = alloca <2 x ptr>, align 1
-; CGSCC-NEXT: [[G:%.*]] = getelementptr i8, ptr [[P_ADDR]], i32 [[I]]
+; CGSCC-NEXT: [[P_ADDR1:%.*]] = alloca i8, i32 16, align 1
+; CGSCC-NEXT: [[G:%.*]] = getelementptr i8, ptr [[P_ADDR1]], i32 [[I]]
+; CGSCC-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[G]] to i32
+; CGSCC-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], 2147483639
+; CGSCC-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr
+; CGSCC-NEXT: store ptr [[P]], ptr [[TMP3]], align 8
; CGSCC-NEXT: store ptr [[P]], ptr [[G]], align 1
-; CGSCC-NEXT: [[R:%.*]] = load ptr, ptr [[P_ADDR]], align 1
+; CGSCC-NEXT: [[R:%.*]] = load ptr, ptr [[P_ADDR1]], align 1
; CGSCC-NEXT: ret ptr [[R]]
;
%p.addr = alloca <2 x ptr>, align 1
@@ -782,16 +790,24 @@ define void @b64613_positive(ptr noundef %p, i32 %i) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define void @b64613_positive
; TUNIT-SAME: (ptr nocapture nofree noundef [[P:%.*]], i32 [[I:%.*]]) #[[ATTR11]] {
-; TUNIT-NEXT: [[P_ADDR:%.*]] = alloca <2 x ptr>, align 1
-; TUNIT-NEXT: [[G:%.*]] = getelementptr i8, ptr [[P_ADDR]], i32 [[I]]
+; TUNIT-NEXT: [[P_ADDR1:%.*]] = alloca i8, i32 16, align 1
+; TUNIT-NEXT: [[G:%.*]] = getelementptr i8, ptr [[P_ADDR1]], i32 [[I]]
+; TUNIT-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[G]] to i32
+; TUNIT-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], 2147483639
+; TUNIT-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr
+; TUNIT-NEXT: store ptr [[P]], ptr [[TMP3]], align 8
; TUNIT-NEXT: store ptr [[P]], ptr [[G]], align 1
; TUNIT-NEXT: ret void
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define void @b64613_positive
; CGSCC-SAME: (ptr nocapture nofree noundef [[P:%.*]], i32 [[I:%.*]]) #[[ATTR13:[0-9]+]] {
-; CGSCC-NEXT: [[P_ADDR:%.*]] = alloca <2 x ptr>, align 1
-; CGSCC-NEXT: [[G:%.*]] = getelementptr i8, ptr [[P_ADDR]], i32 [[I]]
+; CGSCC-NEXT: [[P_ADDR1:%.*]] = alloca i8, i32 16, align 1
+; CGSCC-NEXT: [[G:%.*]] = getelementptr i8, ptr [[P_ADDR1]], i32 [[I]]
+; CGSCC-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[G]] to i32
+; CGSCC-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], 2147483639
+; CGSCC-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr
+; CGSCC-NEXT: store ptr [[P]], ptr [[TMP3]], align 8
; CGSCC-NEXT: store ptr [[P]], ptr [[G]], align 1
; CGSCC-NEXT: ret void
;
diff --git a/llvm/test/Transforms/Attributor/nofpclass.ll b/llvm/test/Transforms/Attributor/nofpclass.ll
index b38f9bae50ccc..caa784b85d3d9 100644
--- a/llvm/test/Transforms/Attributor/nofpclass.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-attributes --check-globals --version 2
; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,TUNIT
; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,CGSCC
@@ -687,9 +687,9 @@ define float @pass_nofpclass_inf_through_memory(float nofpclass(inf) %arg) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define float @pass_nofpclass_inf_through_memory
; TUNIT-SAME: (float nofpclass(inf) [[ARG:%.*]]) #[[ATTR3]] {
-; TUNIT-NEXT: [[ALLOCA:%.*]] = alloca float, align 4
-; TUNIT-NEXT: store float [[ARG]], ptr [[ALLOCA]], align 4
-; TUNIT-NEXT: [[RET:%.*]] = call float @returned_load(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ALLOCA]]) #[[ATTR21:[0-9]+]]
+; TUNIT-NEXT: [[ALLOCA1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: store float [[ARG]], ptr [[ALLOCA1]], align 4
+; TUNIT-NEXT: [[RET:%.*]] = call float @returned_load(ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[ALLOCA1]]) #[[ATTR21:[0-9]+]]
; TUNIT-NEXT: ret float [[RET]]
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
@@ -2520,15 +2520,15 @@ define float @call_through_memory0(float nofpclass(nan) %val) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define nofpclass(nan) float @call_through_memory0
; TUNIT-SAME: (float returned nofpclass(nan) [[VAL:%.*]]) #[[ATTR3]] {
-; TUNIT-NEXT: [[ALLOCA:%.*]] = alloca float, align 4
-; TUNIT-NEXT: store float [[VAL]], ptr [[ALLOCA]], align 4
+; TUNIT-NEXT: [[ALLOCA1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: store float [[VAL]], ptr [[ALLOCA1]], align 4
; TUNIT-NEXT: ret float [[VAL]]
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define float @call_through_memory0
; CGSCC-SAME: (float nofpclass(nan) [[VAL:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT: [[ALLOCA:%.*]] = alloca float, align 4
-; CGSCC-NEXT: store float [[VAL]], ptr [[ALLOCA]], align 4
+; CGSCC-NEXT: [[ALLOCA1:%.*]] = alloca i8, i32 4, align 4
+; CGSCC-NEXT: store float [[VAL]], ptr [[ALLOCA1]], align 4
; CGSCC-NEXT: [[THROUGH_MEMORY:%.*]] = call float @through_memory0(float nofpclass(nan) [[VAL]]) #[[ATTR19]]
; CGSCC-NEXT: ret float [[THROUGH_MEMORY]]
;
@@ -2542,17 +2542,17 @@ define float @call_through_memory1(float nofpclass(nan) %val) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define float @call_through_memory1
; TUNIT-SAME: (float nofpclass(nan) [[VAL:%.*]]) #[[ATTR3]] {
-; TUNIT-NEXT: [[ALLOCA:%.*]] = alloca float, align 4
-; TUNIT-NEXT: store float [[VAL]], ptr [[ALLOCA]], align 4
-; TUNIT-NEXT: [[TMP1:%.*]] = load float, ptr [[ALLOCA]], align 4
+; TUNIT-NEXT: [[ALLOCA1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: store float [[VAL]], ptr [[ALLOCA1]], align 4
+; TUNIT-NEXT: [[TMP1:%.*]] = load float, ptr [[ALLOCA1]], align 4
; TUNIT-NEXT: [[THROUGH_MEMORY:%.*]] = call float @through_memory1(float [[TMP1]]) #[[ATTR21]]
; TUNIT-NEXT: ret float [[THROUGH_MEMORY]]
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define float @call_through_memory1
; CGSCC-SAME: (float nofpclass(nan) [[VAL:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT: [[ALLOCA:%.*]] = alloca float, align 4
-; CGSCC-NEXT: store float [[VAL]], ptr [[ALLOCA]], align 4
+; CGSCC-NEXT: [[ALLOCA1:%.*]] = alloca i8, i32 4, align 4
+; CGSCC-NEXT: store float [[VAL]], ptr [[ALLOCA1]], align 4
; CGSCC-NEXT: [[THROUGH_MEMORY:%.*]] = call float @through_memory1(float nofpclass(nan) [[VAL]]) #[[ATTR19]]
; CGSCC-NEXT: ret float [[THROUGH_MEMORY]]
;
@@ -2565,16 +2565,16 @@ define float @call_through_memory1(float nofpclass(nan) %val) {
define float @call_through_memory2(float nofpclass(nan) %val) {
; TUNIT-LABEL: define float @call_through_memory2
; TUNIT-SAME: (float nofpclass(nan) [[VAL:%.*]]) {
-; TUNIT-NEXT: [[ALLOCA:%.*]] = alloca float, align 4
-; TUNIT-NEXT: store float [[VAL]], ptr [[ALLOCA]], align 4
-; TUNIT-NEXT: [[TMP1:%.*]] = load float, ptr [[ALLOCA]], align 4
+; TUNIT-NEXT: [[ALLOCA1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: store float [[VAL]], ptr [[ALLOCA1]], align 4
+; TUNIT-NEXT: [[TMP1:%.*]] = load float, ptr [[ALLOCA1]], align 4
; TUNIT-NEXT: [[THROUGH_MEMORY:%.*]] = call float @through_memory2(float [[TMP1]])
; TUNIT-NEXT: ret float [[THROUGH_MEMORY]]
;
; CGSCC-LABEL: define float @call_through_memory2
; CGSCC-SAME: (float nofpclass(nan) [[VAL:%.*]]) {
-; CGSCC-NEXT: [[ALLOCA:%.*]] = alloca float, align 4
-; CGSCC-NEXT: store float [[VAL]], ptr [[ALLOCA]], align 4
+; CGSCC-NEXT: [[ALLOCA1:%.*]] = alloca i8, i32 4, align 4
+; CGSCC-NEXT: store float [[VAL]], ptr [[ALLOCA1]], align 4
; CGSCC-NEXT: [[THROUGH_MEMORY:%.*]] = call float @through_memory2(float nofpclass(nan) [[VAL]])
; CGSCC-NEXT: ret float [[THROUGH_MEMORY]]
;
@@ -2689,3 +2689,51 @@ attributes #2 = { "denormal-fp-math"="ieee,preserve-sign" }
attributes #3 = { "denormal-fp-math"="positive-zero,positive-zero" }
attributes #4 = { "denormal-fp-math"="positive-zero,ieee" }
attributes #5 = { "denormal-fp-math"="ieee,positive-zero" }
+;.
+; TUNIT: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+; TUNIT: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; TUNIT: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind strictfp willreturn memory(inaccessiblemem: readwrite) }
+; TUNIT: attributes #[[ATTR3]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
+; TUNIT: attributes #[[ATTR4]] = { mustprogress nofree nosync nounwind willreturn memory(none) }
+; TUNIT: attributes #[[ATTR5]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read) }
+; TUNIT: attributes #[[ATTR6]] = { mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) }
+; TUNIT: attributes #[[ATTR7]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) }
+; TUNIT: attributes #[[ATTR8]] = { mustprogress nofree norecurse nosync nounwind strictfp willreturn memory(inaccessiblemem: readwrite) }
+; TUNIT: attributes #[[ATTR9]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="positive-zero,positive-zero" }
+; TUNIT: attributes #[[ATTR10]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="preserve-sign,preserve-sign" }
+; TUNIT: attributes #[[ATTR11]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="ieee,preserve-sign" }
+; TUNIT: attributes #[[ATTR12]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="positive-zero,ieee" }
+; TUNIT: attributes #[[ATTR13]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="preserve-sign,ieee" }
+; TUNIT: attributes #[[ATTR14]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="ieee,positive-zero" }
+; TUNIT: attributes #[[ATTR15]] = { memory(readwrite, argmem: none) }
+; TUNIT: attributes #[[ATTR16]] = { norecurse }
+; TUNIT: attributes #[[ATTR17]] = { nounwind willreturn }
+; TUNIT: attributes #[[ATTR18]] = { nofree willreturn memory(write) }
+; TUNIT: attributes #[[ATTR19]] = { nofree nosync nounwind willreturn memory(none) }
+; TUNIT: attributes #[[ATTR20]] = { nofree willreturn }
+; TUNIT: attributes #[[ATTR21]] = { nofree nosync nounwind willreturn memory(read) }
+; TUNIT: attributes #[[ATTR22]] = { nofree nosync willreturn }
+;.
+; CGSCC: attributes #[[ATTR0:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+; CGSCC: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+; CGSCC: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind strictfp willreturn memory(inaccessiblemem: readwrite) }
+; CGSCC: attributes #[[ATTR3]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
+; CGSCC: attributes #[[ATTR4]] = { mustprogress nofree nosync nounwind willreturn memory(none) }
+; CGSCC: attributes #[[ATTR5]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: read) }
+; CGSCC: attributes #[[ATTR6]] = { mustprogress nofree norecurse nounwind willreturn memory(argmem: readwrite) }
+; CGSCC: attributes #[[ATTR7]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(argmem: readwrite) }
+; CGSCC: attributes #[[ATTR8]] = { mustprogress nofree norecurse nosync nounwind strictfp willreturn memory(inaccessiblemem: readwrite) }
+; CGSCC: attributes #[[ATTR9]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="positive-zero,positive-zero" }
+; CGSCC: attributes #[[ATTR10]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="preserve-sign,preserve-sign" }
+; CGSCC: attributes #[[ATTR11]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="ieee,preserve-sign" }
+; CGSCC: attributes #[[ATTR12]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="positive-zero,ieee" }
+; CGSCC: attributes #[[ATTR13]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="preserve-sign,ieee" }
+; CGSCC: attributes #[[ATTR14]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) "denormal-fp-math"="ieee,positive-zero" }
+; CGSCC: attributes #[[ATTR15]] = { memory(readwrite, argmem: none) }
+; CGSCC: attributes #[[ATTR16]] = { norecurse }
+; CGSCC: attributes #[[ATTR17]] = { nounwind willreturn }
+; CGSCC: attributes #[[ATTR18]] = { nofree willreturn memory(write) }
+; CGSCC: attributes #[[ATTR19]] = { nofree nosync willreturn }
+; CGSCC: attributes #[[ATTR20]] = { nofree willreturn }
+; CGSCC: attributes #[[ATTR21]] = { nofree willreturn memory(read) }
+;.
diff --git a/llvm/test/Transforms/Attributor/norecurse.ll b/llvm/test/Transforms/Attributor/norecurse.ll
index f139f193f10de..f2730594b2c16 100644
--- a/llvm/test/Transforms/Attributor/norecurse.ll
+++ b/llvm/test/Transforms/Attributor/norecurse.ll
@@ -159,8 +159,8 @@ define void @f(i32 %x) {
; TUNIT-LABEL: define {{[^@]+}}@f
; TUNIT-SAME: (i32 [[X:%.*]]) #[[ATTR1]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[X_ADDR:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: store i32 [[X]], ptr [[X_ADDR]], align 4
+; TUNIT-NEXT: [[X_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: store i32 [[X]], ptr [[X_ADDR1]], align 4
; TUNIT-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[X]], 0
; TUNIT-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
; TUNIT: if.then:
@@ -172,7 +172,7 @@ define void @f(i32 %x) {
; CGSCC-LABEL: define {{[^@]+}}@f
; CGSCC-SAME: (i32 [[X:%.*]]) #[[ATTR0]] {
; CGSCC-NEXT: entry:
-; CGSCC-NEXT: [[X_ADDR:%.*]] = alloca i32, align 4
+; CGSCC-NEXT: [[X_ADDR12:%.*]] = alloca i8, i32 4, align 4
; CGSCC-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[X]], 0
; CGSCC-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
; CGSCC: if.then:
diff --git a/llvm/test/Transforms/Attributor/openmp_parallel.ll b/llvm/test/Transforms/Attributor/openmp_parallel.ll
index 02636ab926dde..aecedbe8ac34f 100644
--- a/llvm/test/Transforms/Attributor/openmp_parallel.ll
+++ b/llvm/test/Transforms/Attributor/openmp_parallel.ll
@@ -23,12 +23,12 @@ define dso_local void @func(ptr nocapture %a, ptr %b, i32 %N) local_unnamed_addr
; TUNIT-LABEL: define {{[^@]+}}@func
; TUNIT-SAME: (ptr nocapture nofree writeonly [[A:%.*]], ptr nocapture nofree readonly [[B:%.*]], i32 [[N:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8
-; TUNIT-NEXT: [[B_ADDR:%.*]] = alloca ptr, align 8
-; TUNIT-NEXT: [[N_ADDR:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: store ptr [[A]], ptr [[A_ADDR]], align 8
-; TUNIT-NEXT: store ptr [[B]], ptr [[B_ADDR]], align 8
-; TUNIT-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB2]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(4) undef, ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[A_ADDR]], ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[B_ADDR]])
+; TUNIT-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 8, align 8
+; TUNIT-NEXT: [[B_ADDR2:%.*]] = alloca i8, i32 8, align 8
+; TUNIT-NEXT: [[N_ADDR3:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: store ptr [[A]], ptr [[A_ADDR1]], align 8
+; TUNIT-NEXT: store ptr [[B]], ptr [[B_ADDR2]], align 8
+; TUNIT-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB2]], i32 noundef 3, ptr noundef nonnull @.omp_outlined., ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(4) undef, ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[A_ADDR1]], ptr noalias nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[B_ADDR2]])
; TUNIT-NEXT: ret void
;
; CGSCC: Function Attrs: nounwind uwtable
diff --git a/llvm/test/Transforms/Attributor/pointer-info.ll b/llvm/test/Transforms/Attributor/pointer-info.ll
index 6afdbdaee317c..9406c7b2efc48 100644
--- a/llvm/test/Transforms/Attributor/pointer-info.ll
+++ b/llvm/test/Transforms/Attributor/pointer-info.ll
@@ -10,10 +10,14 @@ define void @foo(ptr %ptr) {
; TUNIT-LABEL: define {{[^@]+}}@foo
; TUNIT-SAME: (ptr nocapture nofree readnone [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[TMP0:%.*]] = alloca [[STRUCT_TEST_A:%.*]], align 8
+; TUNIT-NEXT: [[TMP0:%.*]] = alloca i8, i32 8, align 8
; TUNIT-NEXT: br label [[CALL_BR:%.*]]
; TUNIT: call.br:
-; TUNIT-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_TEST_A]], ptr [[TMP0]], i64 0, i32 2
+; TUNIT-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_TEST_A:%.*]], ptr [[TMP0]], i64 0, i32 2
+; TUNIT-NEXT: [[TMP2:%.*]] = ptrtoint ptr [[TMP1]] to i32
+; TUNIT-NEXT: [[TMP3:%.*]] = sub i32 [[TMP2]], 16
+; TUNIT-NEXT: [[TMP4:%.*]] = inttoptr i32 [[TMP3]] to ptr
+; TUNIT-NEXT: store ptr [[PTR]], ptr [[TMP4]], align 8
; TUNIT-NEXT: tail call void @bar(ptr noalias nocapture nofree noundef nonnull readonly byval([[STRUCT_TEST_A]]) align 8 dereferenceable(24) [[TMP0]]) #[[ATTR2:[0-9]+]]
; TUNIT-NEXT: ret void
;
diff --git a/llvm/test/Transforms/Attributor/value-simplify-assume.ll b/llvm/test/Transforms/Attributor/value-simplify-assume.ll
index b01a43e3ec758..d3e67b0cee80f 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-assume.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-assume.ll
@@ -120,19 +120,19 @@ define i1 @keep_assume_4c_nr() norecurse {
; TUNIT: Function Attrs: norecurse
; TUNIT-LABEL: define {{[^@]+}}@keep_assume_4c_nr
; TUNIT-SAME: () #[[ATTR2]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, i32 -2147483648, align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef true) #[[ATTR7]]
-; TUNIT-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; TUNIT-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK1]])
; TUNIT-NEXT: ret i1 true
;
; CGSCC: Function Attrs: norecurse
; CGSCC-LABEL: define {{[^@]+}}@keep_assume_4c_nr
; CGSCC-SAME: () #[[ATTR2]] {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; CGSCC-NEXT: store i1 true, ptr [[STACK]], align 1
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, i32 -2147483648, align 1
+; CGSCC-NEXT: store i1 true, ptr [[STACK1]], align 1
; CGSCC-NEXT: call void @llvm.assume(i1 noundef true) #[[ATTR8]]
-; CGSCC-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; CGSCC-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK1]])
; CGSCC-NEXT: ret i1 true
;
%stack = alloca i1
@@ -166,16 +166,16 @@ define i1 @drop_assume_1_nr(i1 %arg) norecurse {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@drop_assume_1_nr
; TUNIT-SAME: (i1 returned [[ARG:%.*]]) #[[ATTR4:[0-9]+]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
+; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR7]]
; TUNIT-NEXT: ret i1 [[ARG]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; CGSCC-LABEL: define {{[^@]+}}@drop_assume_1_nr
; CGSCC-SAME: (i1 returned [[ARG:%.*]]) #[[ATTR4:[0-9]+]] {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, align 1
+; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
; CGSCC-NEXT: call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR8]]
; CGSCC-NEXT: ret i1 [[ARG]]
;
@@ -244,19 +244,19 @@ define i1 @keep_assume_4_nr(i1 %arg) norecurse {
; TUNIT: Function Attrs: norecurse
; TUNIT-LABEL: define {{[^@]+}}@keep_assume_4_nr
; TUNIT-SAME: (i1 returned [[ARG:%.*]]) #[[ATTR2]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, i32 -2147483648, align 1
+; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR7]]
-; TUNIT-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; TUNIT-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK1]])
; TUNIT-NEXT: ret i1 [[ARG]]
;
; CGSCC: Function Attrs: norecurse
; CGSCC-LABEL: define {{[^@]+}}@keep_assume_4_nr
; CGSCC-SAME: (i1 returned [[ARG:%.*]]) #[[ATTR2]] {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, i32 -2147483648, align 1
+; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
; CGSCC-NEXT: call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR8]]
-; CGSCC-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; CGSCC-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK1]])
; CGSCC-NEXT: ret i1 [[ARG]]
;
%stack = alloca i1
@@ -271,8 +271,8 @@ define i1 @assume_1_nr(i1 %arg, i1 %cond) norecurse {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_1_nr
; TUNIT-SAME: (i1 returned [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
+; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR7]]
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
@@ -285,8 +285,8 @@ define i1 @assume_1_nr(i1 %arg, i1 %cond) norecurse {
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; CGSCC-LABEL: define {{[^@]+}}@assume_1_nr
; CGSCC-SAME: (i1 returned [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, align 1
+; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
; CGSCC-NEXT: call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR8]]
; CGSCC-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; CGSCC: t:
@@ -342,32 +342,32 @@ define i1 @assume_2_nr(i1 %arg, i1 %cond) norecurse {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_2_nr
; TUNIT-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store i1 false, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 false, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M]]
; TUNIT: m:
-; TUNIT-NEXT: [[L:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[L:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L]]) #[[ATTR7]]
; TUNIT-NEXT: ret i1 [[L]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; CGSCC-LABEL: define {{[^@]+}}@assume_2_nr
; CGSCC-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; CGSCC-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; CGSCC: t:
-; CGSCC-NEXT: store i1 true, ptr [[STACK]], align 1
+; CGSCC-NEXT: store i1 true, ptr [[STACK1]], align 1
; CGSCC-NEXT: br label [[M:%.*]]
; CGSCC: f:
-; CGSCC-NEXT: store i1 false, ptr [[STACK]], align 1
+; CGSCC-NEXT: store i1 false, ptr [[STACK1]], align 1
; CGSCC-NEXT: br label [[M]]
; CGSCC: m:
-; CGSCC-NEXT: [[L:%.*]] = load i1, ptr [[STACK]], align 1
+; CGSCC-NEXT: [[L:%.*]] = load i1, ptr [[STACK1]], align 1
; CGSCC-NEXT: call void @llvm.assume(i1 noundef [[L]]) #[[ATTR8]]
; CGSCC-NEXT: ret i1 [[L]]
;
@@ -390,14 +390,14 @@ define void @assume_2b_nr(i1 %arg, i1 %cond) norecurse {
; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; CHECK-LABEL: define {{[^@]+}}@assume_2b_nr
; CHECK-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; CHECK-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; CHECK-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; CHECK: t:
; CHECK-NEXT: br label [[M:%.*]]
; CHECK: f:
; CHECK-NEXT: br label [[M]]
; CHECK: m:
-; CHECK-NEXT: [[L:%.*]] = load i1, ptr [[STACK]], align 1
+; CHECK-NEXT: [[L:%.*]] = load i1, ptr [[STACK1]], align 1
; CHECK-NEXT: ret void
;
%stack = alloca i1
@@ -419,16 +419,16 @@ define i1 @assume_3_nr(i1 %arg, i1 %cond) norecurse {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_3_nr
; TUNIT-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store i1 false, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 false, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M]]
; TUNIT: m:
-; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8:[0-9]+]]
+; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK1]]) #[[ATTR8:[0-9]+]]
; TUNIT-NEXT: ret i1 [[R]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -469,18 +469,18 @@ define i1 @assume_4_nr(i1 %arg, i1 %cond) norecurse {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_4_nr
; TUNIT-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store i1 false, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 false, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M]]
; TUNIT: m:
-; TUNIT-NEXT: [[L:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[L:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L]]) #[[ATTR7]]
-; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK1]]) #[[ATTR8]]
; TUNIT-NEXT: ret i1 [[R]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -521,22 +521,22 @@ define i1 @assume_5_nr(i1 %arg, i1 %cond) norecurse {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_5_nr
; TUNIT-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
-; TUNIT-NEXT: [[L2:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
+; TUNIT-NEXT: [[L2:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L2]]) #[[ATTR7]]
; TUNIT-NEXT: br label [[M:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store i1 false, ptr [[STACK]], align 1
-; TUNIT-NEXT: [[L3:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 false, ptr [[STACK1]], align 1
+; TUNIT-NEXT: [[L3:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L3]]) #[[ATTR7]]
; TUNIT-NEXT: br label [[M]]
; TUNIT: m:
-; TUNIT-NEXT: [[L4:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[L4:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR9:[0-9]+]]
-; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK1]]) #[[ATTR8]]
; TUNIT-NEXT: ret i1 [[R]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -589,23 +589,23 @@ define i1 @assume_5c_nr(i1 %cond) norecurse {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_5c_nr
; TUNIT-SAME: (i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef true) #[[ATTR7]]
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
-; TUNIT-NEXT: [[L2:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
+; TUNIT-NEXT: [[L2:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L2]]) #[[ATTR7]]
; TUNIT-NEXT: br label [[M:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store i1 false, ptr [[STACK]], align 1
-; TUNIT-NEXT: [[L3:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 false, ptr [[STACK1]], align 1
+; TUNIT-NEXT: [[L3:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L3]]) #[[ATTR7]]
; TUNIT-NEXT: br label [[M]]
; TUNIT: m:
-; TUNIT-NEXT: [[L4:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[L4:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR9]]
-; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK1]]) #[[ATTR8]]
; TUNIT-NEXT: ret i1 [[R]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -741,19 +741,19 @@ define i1 @keep_assume_3c() {
define i1 @keep_assume_4c() {
;
; TUNIT-LABEL: define {{[^@]+}}@keep_assume_4c() {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
-; TUNIT-NEXT: [[L4:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, i32 -2147483648, align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
+; TUNIT-NEXT: [[L4:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR7]]
-; TUNIT-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; TUNIT-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK1]])
; TUNIT-NEXT: ret i1 [[L4]]
;
; CGSCC-LABEL: define {{[^@]+}}@keep_assume_4c() {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; CGSCC-NEXT: store i1 true, ptr [[STACK]], align 1
-; CGSCC-NEXT: [[L4:%.*]] = load i1, ptr [[STACK]], align 1
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, i32 -2147483648, align 1
+; CGSCC-NEXT: store i1 true, ptr [[STACK1]], align 1
+; CGSCC-NEXT: [[L4:%.*]] = load i1, ptr [[STACK1]], align 1
; CGSCC-NEXT: call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR8]]
-; CGSCC-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; CGSCC-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK1]])
; CGSCC-NEXT: ret i1 [[L4]]
;
%stack = alloca i1
@@ -786,16 +786,16 @@ define i1 @drop_assume_1(i1 %arg) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@drop_assume_1
; TUNIT-SAME: (i1 returned [[ARG:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
+; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR7]]
; TUNIT-NEXT: ret i1 [[ARG]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; CGSCC-LABEL: define {{[^@]+}}@drop_assume_1
; CGSCC-SAME: (i1 returned [[ARG:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, align 1
+; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
; CGSCC-NEXT: call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR8]]
; CGSCC-NEXT: ret i1 [[ARG]]
;
@@ -860,20 +860,20 @@ define i1 @keep_assume_4(i1 %arg) {
;
; TUNIT-LABEL: define {{[^@]+}}@keep_assume_4
; TUNIT-SAME: (i1 [[ARG:%.*]]) {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
-; TUNIT-NEXT: [[L:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, i32 -2147483648, align 1
+; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
+; TUNIT-NEXT: [[L:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L]]) #[[ATTR7]]
-; TUNIT-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; TUNIT-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK1]])
; TUNIT-NEXT: ret i1 [[L]]
;
; CGSCC-LABEL: define {{[^@]+}}@keep_assume_4
; CGSCC-SAME: (i1 [[ARG:%.*]]) {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
-; CGSCC-NEXT: [[L:%.*]] = load i1, ptr [[STACK]], align 1
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, i32 -2147483648, align 1
+; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
+; CGSCC-NEXT: [[L:%.*]] = load i1, ptr [[STACK1]], align 1
; CGSCC-NEXT: call void @llvm.assume(i1 noundef [[L]]) #[[ATTR8]]
-; CGSCC-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK]])
+; CGSCC-NEXT: call void @useI1p(ptr noalias nocapture noundef nonnull dereferenceable(1) [[STACK1]])
; CGSCC-NEXT: ret i1 [[L]]
;
%stack = alloca i1
@@ -888,8 +888,8 @@ define i1 @assume_1(i1 %arg, i1 %cond) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_1
; TUNIT-SAME: (i1 returned [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
+; TUNIT-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR7]]
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
@@ -902,8 +902,8 @@ define i1 @assume_1(i1 %arg, i1 %cond) {
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; CGSCC-LABEL: define {{[^@]+}}@assume_1
; CGSCC-SAME: (i1 returned [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i1, align 1
-; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK]], align 1
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, align 1
+; CGSCC-NEXT: store i1 [[ARG]], ptr [[STACK1]], align 1
; CGSCC-NEXT: call void @llvm.assume(i1 noundef [[ARG]]) #[[ATTR8]]
; CGSCC-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; CGSCC: t:
@@ -959,32 +959,32 @@ define i1 @assume_2(i1 %arg, i1 %cond) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_2
; TUNIT-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store i1 false, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 false, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M]]
; TUNIT: m:
-; TUNIT-NEXT: [[L:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[L:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L]]) #[[ATTR7]]
; TUNIT-NEXT: ret i1 [[L]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; CGSCC-LABEL: define {{[^@]+}}@assume_2
; CGSCC-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; CGSCC-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; CGSCC: t:
-; CGSCC-NEXT: store i1 true, ptr [[STACK]], align 1
+; CGSCC-NEXT: store i1 true, ptr [[STACK1]], align 1
; CGSCC-NEXT: br label [[M:%.*]]
; CGSCC: f:
-; CGSCC-NEXT: store i1 false, ptr [[STACK]], align 1
+; CGSCC-NEXT: store i1 false, ptr [[STACK1]], align 1
; CGSCC-NEXT: br label [[M]]
; CGSCC: m:
-; CGSCC-NEXT: [[L:%.*]] = load i1, ptr [[STACK]], align 1
+; CGSCC-NEXT: [[L:%.*]] = load i1, ptr [[STACK1]], align 1
; CGSCC-NEXT: call void @llvm.assume(i1 noundef [[L]]) #[[ATTR8]]
; CGSCC-NEXT: ret i1 [[L]]
;
@@ -1007,14 +1007,14 @@ define void @assume_2b(i1 %arg, i1 %cond) {
; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; CHECK-LABEL: define {{[^@]+}}@assume_2b
; CHECK-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; CHECK-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; CHECK-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; CHECK-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; CHECK: t:
; CHECK-NEXT: br label [[M:%.*]]
; CHECK: f:
; CHECK-NEXT: br label [[M]]
; CHECK: m:
-; CHECK-NEXT: [[L:%.*]] = load i1, ptr [[STACK]], align 1
+; CHECK-NEXT: [[L:%.*]] = load i1, ptr [[STACK1]], align 1
; CHECK-NEXT: ret void
;
%stack = alloca i1
@@ -1036,16 +1036,16 @@ define i1 @assume_3(i1 %arg, i1 %cond) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_3
; TUNIT-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store i1 false, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 false, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M]]
; TUNIT: m:
-; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK1]]) #[[ATTR8]]
; TUNIT-NEXT: ret i1 [[R]]
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -1086,18 +1086,18 @@ define i1 @assume_4(i1 %arg, i1 %cond) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_4
; TUNIT-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store i1 false, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 false, ptr [[STACK1]], align 1
; TUNIT-NEXT: br label [[M]]
; TUNIT: m:
-; TUNIT-NEXT: [[L:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[L:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L]]) #[[ATTR7]]
-; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK1]]) #[[ATTR8]]
; TUNIT-NEXT: ret i1 [[R]]
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -1138,22 +1138,22 @@ define i1 @assume_5(i1 %arg, i1 %cond) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_5
; TUNIT-SAME: (i1 [[ARG:%.*]], i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
-; TUNIT-NEXT: [[L2:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
+; TUNIT-NEXT: [[L2:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L2]]) #[[ATTR7]]
; TUNIT-NEXT: br label [[M:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store i1 false, ptr [[STACK]], align 1
-; TUNIT-NEXT: [[L3:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 false, ptr [[STACK1]], align 1
+; TUNIT-NEXT: [[L3:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L3]]) #[[ATTR7]]
; TUNIT-NEXT: br label [[M]]
; TUNIT: m:
-; TUNIT-NEXT: [[L4:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[L4:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR9]]
-; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK1]]) #[[ATTR8]]
; TUNIT-NEXT: ret i1 [[R]]
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
@@ -1206,23 +1206,23 @@ define i1 @assume_5c(i1 %cond) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(inaccessiblemem: readwrite)
; TUNIT-LABEL: define {{[^@]+}}@assume_5c
; TUNIT-SAME: (i1 noundef [[COND:%.*]]) #[[ATTR4]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i1, align 1
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef true) #[[ATTR7]]
; TUNIT-NEXT: br i1 [[COND]], label [[T:%.*]], label [[F:%.*]]
; TUNIT: t:
-; TUNIT-NEXT: store i1 true, ptr [[STACK]], align 1
-; TUNIT-NEXT: [[L2:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 true, ptr [[STACK1]], align 1
+; TUNIT-NEXT: [[L2:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L2]]) #[[ATTR7]]
; TUNIT-NEXT: br label [[M:%.*]]
; TUNIT: f:
-; TUNIT-NEXT: store i1 false, ptr [[STACK]], align 1
-; TUNIT-NEXT: [[L3:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: store i1 false, ptr [[STACK1]], align 1
+; TUNIT-NEXT: [[L3:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L3]]) #[[ATTR7]]
; TUNIT-NEXT: br label [[M]]
; TUNIT: m:
-; TUNIT-NEXT: [[L4:%.*]] = load i1, ptr [[STACK]], align 1
+; TUNIT-NEXT: [[L4:%.*]] = load i1, ptr [[STACK1]], align 1
; TUNIT-NEXT: call void @llvm.assume(i1 noundef [[L4]]) #[[ATTR9]]
-; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK]]) #[[ATTR8]]
+; TUNIT-NEXT: [[R:%.*]] = call i1 @readI1p(ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[STACK1]]) #[[ATTR8]]
; TUNIT-NEXT: ret i1 [[R]]
;
; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(inaccessiblemem: readwrite)
diff --git a/llvm/test/Transforms/Attributor/value-simplify-dominance.ll b/llvm/test/Transforms/Attributor/value-simplify-dominance.ll
index 7d95f35b24fd0..a0409eb4a83ca 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-dominance.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-dominance.ll
@@ -8,7 +8,7 @@ define i32 @many_writes_nosycn(i1 %c0, i1 %c1, i1 %c2) nosync {
; CHECK: Function Attrs: norecurse nosync
; CHECK-LABEL: define {{[^@]+}}@many_writes_nosycn
; CHECK-SAME: (i1 [[C0:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) #[[ATTR1:[0-9]+]] {
-; CHECK-NEXT: [[P:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[P1:%.*]] = alloca i8, i32 4, align 4
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: br i1 [[C0]], label [[T0:%.*]], label [[F0:%.*]]
; CHECK: t0:
@@ -19,19 +19,19 @@ define i32 @many_writes_nosycn(i1 %c0, i1 %c1, i1 %c2) nosync {
; CHECK-NEXT: br i1 [[C2]], label [[F1:%.*]], label [[M1]]
; CHECK: t1:
; CHECK-NEXT: call void @unknown()
-; CHECK-NEXT: store i32 7, ptr [[P]], align 4
+; CHECK-NEXT: store i32 7, ptr [[P1]], align 4
; CHECK-NEXT: br label [[M2:%.*]]
; CHECK: f1:
; CHECK-NEXT: call void @unknown()
-; CHECK-NEXT: store i32 9, ptr [[P]], align 4
+; CHECK-NEXT: store i32 9, ptr [[P1]], align 4
; CHECK-NEXT: br label [[M2]]
; CHECK: m1:
; CHECK-NEXT: call void @unknown()
-; CHECK-NEXT: store i32 11, ptr [[P]], align 4
+; CHECK-NEXT: store i32 11, ptr [[P1]], align 4
; CHECK-NEXT: br label [[M2]]
; CHECK: m2:
; CHECK-NEXT: call void @unknown()
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[P1]], align 4
; CHECK-NEXT: ret i32 [[L]]
;
%p = alloca i32
@@ -74,7 +74,7 @@ define i32 @many_writes(i1 %c0, i1 %c1, i1 %c2) {
; CHECK: Function Attrs: norecurse
; CHECK-LABEL: define {{[^@]+}}@many_writes
; CHECK-SAME: (i1 [[C0:%.*]], i1 [[C1:%.*]], i1 [[C2:%.*]]) #[[ATTR2:[0-9]+]] {
-; CHECK-NEXT: [[P:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[P1:%.*]] = alloca i8, i32 4, align 4
; CHECK-NEXT: call void @unknown()
; CHECK-NEXT: br i1 [[C0]], label [[T0:%.*]], label [[F0:%.*]]
; CHECK: t0:
@@ -85,19 +85,19 @@ define i32 @many_writes(i1 %c0, i1 %c1, i1 %c2) {
; CHECK-NEXT: br i1 [[C2]], label [[F1:%.*]], label [[M1]]
; CHECK: t1:
; CHECK-NEXT: call void @unknown()
-; CHECK-NEXT: store i32 7, ptr [[P]], align 4
+; CHECK-NEXT: store i32 7, ptr [[P1]], align 4
; CHECK-NEXT: br label [[M2:%.*]]
; CHECK: f1:
; CHECK-NEXT: call void @unknown()
-; CHECK-NEXT: store i32 9, ptr [[P]], align 4
+; CHECK-NEXT: store i32 9, ptr [[P1]], align 4
; CHECK-NEXT: br label [[M2]]
; CHECK: m1:
; CHECK-NEXT: call void @unknown()
-; CHECK-NEXT: store i32 11, ptr [[P]], align 4
+; CHECK-NEXT: store i32 11, ptr [[P1]], align 4
; CHECK-NEXT: br label [[M2]]
; CHECK: m2:
; CHECK-NEXT: call void @unknown()
-; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[P1]], align 4
; CHECK-NEXT: ret i32 [[L]]
;
%p = alloca i32
@@ -163,8 +163,8 @@ define i32 @local_stack_remote_write_and_read() norecurse {
; TUNIT: Function Attrs: norecurse
; TUNIT-LABEL: define {{[^@]+}}@local_stack_remote_write_and_read
; TUNIT-SAME: () #[[ATTR2]] {
-; TUNIT-NEXT: [[A:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: [[R:%.*]] = call i32 @remote_write_and_read(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A]])
+; TUNIT-NEXT: [[A1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: [[R:%.*]] = call i32 @remote_write_and_read(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[A1]])
; TUNIT-NEXT: ret i32 42
;
; CGSCC: Function Attrs: norecurse
diff --git a/llvm/test/Transforms/Attributor/value-simplify-gpu.ll b/llvm/test/Transforms/Attributor/value-simplify-gpu.ll
index 04ba6e2dc5f90..1f614f2043d9b 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-gpu.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-gpu.ll
@@ -205,8 +205,8 @@ define internal void @level1(i32 %C) {
; TUNIT-LABEL: define {{[^@]+}}@level1
; TUNIT-SAME: (i32 [[C:%.*]]) #[[ATTR1]] {
; TUNIT-NEXT: entry:
-; TUNIT-NEXT: [[LOCAL:%.*]] = alloca i32, align 4
-; TUNIT-NEXT: call void @level2all_early(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[LOCAL]]) #[[ATTR4]]
+; TUNIT-NEXT: [[LOCAL1:%.*]] = alloca i8, i32 4, align 4
+; TUNIT-NEXT: call void @level2all_early(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[LOCAL1]]) #[[ATTR4]]
; TUNIT-NEXT: [[TOBOOL:%.*]] = icmp ne i32 [[C]], 0
; TUNIT-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
; TUNIT: if.then:
@@ -216,7 +216,7 @@ define internal void @level1(i32 %C) {
; TUNIT-NEXT: call void @level2b() #[[ATTR5]]
; TUNIT-NEXT: br label [[IF_END]]
; TUNIT: if.end:
-; TUNIT-NEXT: call void @level2all_late(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[LOCAL]]) #[[ATTR6]]
+; TUNIT-NEXT: call void @level2all_late(ptr noalias nocapture nofree noundef nonnull writeonly align 4 dereferenceable(4) [[LOCAL1]]) #[[ATTR6]]
; TUNIT-NEXT: ret void
;
; CGSCC: Function Attrs: norecurse nosync nounwind
diff --git a/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll b/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
index 20b52c3fcd85a..ed56bfa4f575d 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
@@ -375,13 +375,13 @@ define dso_local void @spam() {
; TUNIT-LABEL: define {{[^@]+}}@spam
; TUNIT-SAME: () #[[ATTR2:[0-9]+]] {
; TUNIT-NEXT: bb:
-; TUNIT-NEXT: [[TMP:%.*]] = alloca i32, align 4
+; TUNIT-NEXT: [[TMP1:%.*]] = alloca i8, i32 4, align 4
; TUNIT-NEXT: [[X:%.*]] = fptosi float undef to i32
-; TUNIT-NEXT: store i32 [[X]], ptr [[TMP]], align 4
+; TUNIT-NEXT: store i32 [[X]], ptr [[TMP1]], align 4
; TUNIT-NEXT: br label [[BB16:%.*]]
; TUNIT: bb16:
-; TUNIT-NEXT: [[TRUETMP18:%.*]] = icmp eq i32 [[X]], 0
-; TUNIT-NEXT: br i1 [[TRUETMP18]], label [[BB35:%.*]], label [[BB19:%.*]]
+; TUNIT-NEXT: [[TMP18:%.*]] = icmp eq i32 [[X]], 0
+; TUNIT-NEXT: br i1 [[TMP18]], label [[BB35:%.*]], label [[BB19:%.*]]
; TUNIT: bb19:
; TUNIT-NEXT: br label [[BB23:%.*]]
; TUNIT: bb23:
@@ -399,13 +399,13 @@ define dso_local void @spam() {
; CGSCC-LABEL: define {{[^@]+}}@spam
; CGSCC-SAME: () #[[ATTR5:[0-9]+]] {
; CGSCC-NEXT: bb:
-; CGSCC-NEXT: [[TMP:%.*]] = alloca i32, align 4
+; CGSCC-NEXT: [[TMP1:%.*]] = alloca i8, i32 4, align 4
; CGSCC-NEXT: [[X:%.*]] = fptosi float undef to i32
-; CGSCC-NEXT: store i32 [[X]], ptr [[TMP]], align 4
+; CGSCC-NEXT: store i32 [[X]], ptr [[TMP1]], align 4
; CGSCC-NEXT: br label [[BB16:%.*]]
; CGSCC: bb16:
-; CGSCC-NEXT: [[TRUETMP18:%.*]] = icmp eq i32 [[X]], 0
-; CGSCC-NEXT: br i1 [[TRUETMP18]], label [[BB35:%.*]], label [[BB19:%.*]]
+; CGSCC-NEXT: [[TMP18:%.*]] = icmp eq i32 [[X]], 0
+; CGSCC-NEXT: br i1 [[TMP18]], label [[BB35:%.*]], label [[BB19:%.*]]
; CGSCC: bb19:
; CGSCC-NEXT: br label [[BB23:%.*]]
; CGSCC: bb23:
diff --git a/llvm/test/Transforms/Attributor/value-simplify.ll b/llvm/test/Transforms/Attributor/value-simplify.ll
index 62d4f63677df6..b1782bca83dda 100644
--- a/llvm/test/Transforms/Attributor/value-simplify.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify.ll
@@ -1267,21 +1267,21 @@ define internal i8 @memcpy_uses_store(i8 %arg) {
; TUNIT: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; TUNIT-LABEL: define {{[^@]+}}@memcpy_uses_store
; TUNIT-SAME: (i8 [[ARG:%.*]]) #[[ATTR2]] {
-; TUNIT-NEXT: [[SRC:%.*]] = alloca i8, align 1
-; TUNIT-NEXT: [[DST:%.*]] = alloca i8, align 1
-; TUNIT-NEXT: store i8 [[ARG]], ptr [[SRC]], align 1
-; TUNIT-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[DST]], ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[SRC]], i32 noundef 1, i1 noundef false) #[[ATTR12:[0-9]+]]
-; TUNIT-NEXT: [[L:%.*]] = load i8, ptr [[DST]], align 1
+; TUNIT-NEXT: [[SRC1:%.*]] = alloca i8, align 1
+; TUNIT-NEXT: [[DST2:%.*]] = alloca i8, align 1
+; TUNIT-NEXT: store i8 [[ARG]], ptr [[SRC1]], align 1
+; TUNIT-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[DST2]], ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[SRC1]], i32 noundef 1, i1 noundef false) #[[ATTR12:[0-9]+]]
+; TUNIT-NEXT: [[L:%.*]] = load i8, ptr [[DST2]], align 1
; TUNIT-NEXT: ret i8 [[L]]
;
; CGSCC: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; CGSCC-LABEL: define {{[^@]+}}@memcpy_uses_store
; CGSCC-SAME: (i8 [[ARG:%.*]]) #[[ATTR1]] {
-; CGSCC-NEXT: [[SRC:%.*]] = alloca i8, align 1
-; CGSCC-NEXT: [[DST:%.*]] = alloca i8, align 1
-; CGSCC-NEXT: store i8 [[ARG]], ptr [[SRC]], align 1
-; CGSCC-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[DST]], ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[SRC]], i32 noundef 1, i1 noundef false) #[[ATTR16:[0-9]+]]
-; CGSCC-NEXT: [[L:%.*]] = load i8, ptr [[DST]], align 1
+; CGSCC-NEXT: [[SRC1:%.*]] = alloca i8, align 1
+; CGSCC-NEXT: [[DST2:%.*]] = alloca i8, align 1
+; CGSCC-NEXT: store i8 [[ARG]], ptr [[SRC1]], align 1
+; CGSCC-NEXT: call void @llvm.memcpy.p0.p0.i32(ptr noalias nocapture nofree noundef nonnull writeonly dereferenceable(1) [[DST2]], ptr noalias nocapture nofree noundef nonnull readonly dereferenceable(1) [[SRC1]], i32 noundef 1, i1 noundef false) #[[ATTR16:[0-9]+]]
+; CGSCC-NEXT: [[L:%.*]] = load i8, ptr [[DST2]], align 1
; CGSCC-NEXT: ret i8 [[L]]
;
%src = alloca i8
@@ -1316,21 +1316,21 @@ define i32 @test_speculatable_expr() norecurse {
; TUNIT: Function Attrs: norecurse nosync memory(none)
; TUNIT-LABEL: define {{[^@]+}}@test_speculatable_expr
; TUNIT-SAME: () #[[ATTR7:[0-9]+]] {
-; TUNIT-NEXT: [[STACK:%.*]] = alloca i32, align 4
+; TUNIT-NEXT: [[STACK1:%.*]] = alloca i8, i32 4, align 4
; TUNIT-NEXT: [[SPEC_RESULT:%.*]] = call i32 @speculatable() #[[ATTR14:[0-9]+]]
; TUNIT-NEXT: [[PLUS1:%.*]] = add i32 [[SPEC_RESULT]], 1
-; TUNIT-NEXT: store i32 [[PLUS1]], ptr [[STACK]], align 4
-; TUNIT-NEXT: [[TMP1:%.*]] = load i32, ptr [[STACK]], align 4
+; TUNIT-NEXT: store i32 [[PLUS1]], ptr [[STACK1]], align 4
+; TUNIT-NEXT: [[TMP1:%.*]] = load i32, ptr [[STACK1]], align 4
; TUNIT-NEXT: [[RSPEC:%.*]] = call i32 @ret_speculatable_expr(i32 [[TMP1]]) #[[ATTR15:[0-9]+]]
; TUNIT-NEXT: ret i32 [[RSPEC]]
;
; CGSCC: Function Attrs: norecurse nosync memory(none)
; CGSCC-LABEL: define {{[^@]+}}@test_speculatable_expr
; CGSCC-SAME: () #[[ATTR9:[0-9]+]] {
-; CGSCC-NEXT: [[STACK:%.*]] = alloca i32, align 4
+; CGSCC-NEXT: [[STACK1:%.*]] = alloca i8, i32 4, align 4
; CGSCC-NEXT: [[SPEC_RESULT:%.*]] = call i32 @speculatable() #[[ATTR17:[0-9]+]]
; CGSCC-NEXT: [[PLUS1:%.*]] = add i32 [[SPEC_RESULT]], 1
-; CGSCC-NEXT: store i32 [[PLUS1]], ptr [[STACK]], align 4
+; CGSCC-NEXT: store i32 [[PLUS1]], ptr [[STACK1]], align 4
; CGSCC-NEXT: [[RSPEC:%.*]] = call i32 @ret_speculatable_expr(i32 [[PLUS1]]) #[[ATTR17]]
; CGSCC-NEXT: ret i32 [[RSPEC]]
;
diff --git a/llvm/test/Transforms/OpenMP/parallel_deletion.ll b/llvm/test/Transforms/OpenMP/parallel_deletion.ll
index 4619da1206092..ffb86971d1a7e 100644
--- a/llvm/test/Transforms/OpenMP/parallel_deletion.ll
+++ b/llvm/test/Transforms/OpenMP/parallel_deletion.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-attributes --check-globals
; RUN: opt -S -passes='attributor,cgscc(openmp-opt-cgscc)' < %s | FileCheck %s
;
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
@@ -23,10 +23,17 @@ target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16
; }
;
; We delete all but the first of the parallel regions in this test.
+;.
+; CHECK: @[[_STR:[a-zA-Z0-9_$"\\.-]+]] = private unnamed_addr constant [23 x i8] c"
+; CHECK: @[[GLOB0:[0-9]+]] = private unnamed_addr global [[STRUCT_IDENT_T:%.*]] { i32 0, i32 2, i32 0, i32 0, ptr @.str }, align 8
+; CHECK: @[[GLOB1:[0-9]+]] = private unnamed_addr global [[STRUCT_IDENT_T:%.*]] { i32 0, i32 322, i32 0, i32 0, ptr @.str }, align 8
+; CHECK: @[[_GOMP_CRITICAL_USER__REDUCTION_VAR:[a-zA-Z0-9_$"\\.-]+]] = common global [8 x i32] zeroinitializer
+; CHECK: @[[GLOB2:[0-9]+]] = private unnamed_addr global [[STRUCT_IDENT_T:%.*]] { i32 0, i32 18, i32 0, i32 0, ptr @.str }, align 8
+;.
define void @delete_parallel_0() {
; CHECK-LABEL: define {{[^@]+}}@delete_parallel_0() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0:[0-9]+]], i32 noundef 0, ptr noundef nonnull @.omp_outlined.willreturn)
+; CHECK-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 0, ptr noundef nonnull @.omp_outlined.willreturn)
; CHECK-NEXT: ret void
;
; CHECK1-LABEL: define {{[^@]+}}@delete_parallel_0() {
@@ -46,6 +53,7 @@ entry:
}
define internal void @.omp_outlined.willreturn(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
+; CHECK: Function Attrs: mustprogress willreturn
; CHECK-LABEL: define {{[^@]+}}@.omp_outlined.willreturn
; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -68,6 +76,7 @@ entry:
}
define internal void @.omp_outlined.willreturn.0(ptr noalias %.global_tid., ptr noalias %.bound_tid.) willreturn {
+; CHECK: Function Attrs: mustprogress nosync willreturn memory(read)
; CHECK-LABEL: define {{[^@]+}}@.omp_outlined.willreturn.0
; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -90,6 +99,7 @@ entry:
}
define internal void @.omp_outlined.willreturn.1(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
+; CHECK: Function Attrs: mustprogress nosync willreturn memory(none)
; CHECK-LABEL: define {{[^@]+}}@.omp_outlined.willreturn.1
; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR2:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -112,6 +122,7 @@ entry:
}
define internal void @.omp_outlined.willreturn.2(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; CHECK-LABEL: define {{[^@]+}}@.omp_outlined.willreturn.2
; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR3:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -192,6 +203,7 @@ entry:
}
define internal void @.omp_outlined..0(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
+; CHECK: Function Attrs: nosync memory(read)
; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..0
; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR4]] {
; CHECK-NEXT: entry:
@@ -214,6 +226,7 @@ entry:
}
define internal void @.omp_outlined..1(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
+; CHECK: Function Attrs: nosync memory(none)
; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..1
; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR5:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -236,6 +249,7 @@ entry:
}
define internal void @.omp_outlined..2(ptr noalias %.global_tid., ptr noalias %.bound_tid.) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn memory(none)
; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..2
; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]]) #[[ATTR3]] {
; CHECK-NEXT: entry:
@@ -281,14 +295,14 @@ entry:
define void @delete_parallel_2() {
; CHECK-LABEL: define {{[^@]+}}@delete_parallel_2() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull align 4 dereferenceable(4) [[A]]) #[[ATTR18:[0-9]+]]
-; CHECK-NEXT: store i32 0, ptr [[A]], align 4
-; CHECK-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..3, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
-; CHECK-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
-; CHECK-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..5, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A]])
-; CHECK-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..6, ptr nocapture noundef nonnull align 4 dereferenceable(4) [[A]])
-; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[A]])
+; CHECK-NEXT: [[A1:%.*]] = alloca i8, i32 4, align 4
+; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 noundef 4, ptr noundef nonnull align 4 dereferenceable(4) [[A1]]) #[[ATTR18:[0-9]+]]
+; CHECK-NEXT: store i32 0, ptr [[A1]], align 4
+; CHECK-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..3, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A1]])
+; CHECK-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..4, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A1]])
+; CHECK-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..5, ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A1]])
+; CHECK-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB0]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..6, ptr nocapture noundef nonnull align 4 dereferenceable(4) [[A1]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[A1]])
; CHECK-NEXT: ret void
;
; CHECK1-LABEL: define {{[^@]+}}@delete_parallel_2() {
@@ -326,6 +340,7 @@ entry:
}
define internal void @.omp_outlined..3(ptr noalias %.global_tid., ptr noalias %.bound_tid., ptr dereferenceable(4) %a) {
+; CHECK: Function Attrs: nofree nosync nounwind memory(argmem: readwrite, inaccessiblemem: readwrite)
; CHECK-LABEL: define {{[^@]+}}@.omp_outlined..3
; CHECK-SAME: (ptr noalias nocapture nofree readnone [[DOTGLOBAL_TID_:%.*]], ptr noalias nocapture nofree readnone [[DOTBOUND_TID_:%.*]], ptr nocapture nofree noundef nonnull align 4 dereferenceable(4) [[A:%.*]]) #[[ATTR6:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -469,7 +484,7 @@ define internal void @.omp_outlined..5(ptr noalias %.global_tid., ptr noalias %.
; CHECK-NEXT: call void @__kmpc_end_single(ptr noundef nonnull @[[GLOB0]], i32 [[TMP]])
; CHECK-NEXT: br label [[OMP_IF_END]]
; CHECK: omp_if.end:
-; CHECK-NEXT: call void @__kmpc_barrier(ptr noundef nonnull @[[GLOB1:[0-9]+]], i32 [[OMP_GLOBAL_THREAD_NUM]])
+; CHECK-NEXT: call void @__kmpc_barrier(ptr noundef nonnull @[[GLOB1]], i32 [[OMP_GLOBAL_THREAD_NUM]])
; CHECK-NEXT: ret void
;
; CHECK1-LABEL: define {{[^@]+}}@.omp_outlined..5
@@ -535,7 +550,7 @@ define internal void @.omp_outlined..6(ptr noalias %.global_tid., ptr noalias %.
; CHECK-NEXT: store i32 1, ptr [[A1]], align 4
; CHECK-NEXT: store ptr [[A1]], ptr [[DOTOMP_REDUCTION_RED_LIST]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[DOTGLOBAL_TID_]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = call i32 @__kmpc_reduce_nowait(ptr noundef nonnull @[[GLOB2:[0-9]+]], i32 [[TMP2]], i32 noundef 1, i64 noundef 8, ptr noundef nonnull align 8 [[DOTOMP_REDUCTION_RED_LIST]], ptr noundef nonnull @.omp.reduction.reduction_func, ptr noundef nonnull @.gomp_critical_user_.reduction.var)
+; CHECK-NEXT: [[TMP4:%.*]] = call i32 @__kmpc_reduce_nowait(ptr noundef nonnull @[[GLOB2]], i32 [[TMP2]], i32 noundef 1, i64 noundef 8, ptr noundef nonnull align 8 [[DOTOMP_REDUCTION_RED_LIST]], ptr noundef nonnull @.omp.reduction.reduction_func, ptr noundef nonnull @.gomp_critical_user_.reduction.var)
; CHECK-NEXT: switch i32 [[TMP4]], label [[DOTOMP_REDUCTION_DEFAULT:%.*]] [
; CHECK-NEXT: i32 1, label [[DOTOMP_REDUCTION_CASE1:%.*]]
; CHECK-NEXT: i32 2, label [[DOTOMP_REDUCTION_CASE2:%.*]]
@@ -643,6 +658,7 @@ entry:
}
define internal void @.omp.reduction.reduction_func(ptr %arg, ptr %arg1) {
+; CHECK: Function Attrs: mustprogress nofree norecurse nosync nounwind willreturn
; CHECK-LABEL: define {{[^@]+}}@.omp.reduction.reduction_func
; CHECK-SAME: (ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[ARG:%.*]], ptr nocapture nofree noundef nonnull readonly align 8 dereferenceable(8) [[ARG1:%.*]]) #[[ATTR10:[0-9]+]] {
; CHECK-NEXT: entry:
@@ -717,3 +733,30 @@ declare void @readnone() readnone
!6 = !{!"omnipotent char", !7, i64 0}
!7 = !{!"Simple C/C++ TBAA"}
!8 = !{i32 7, !"openmp", i32 50}
+;.
+; CHECK: attributes #[[ATTR0]] = { mustprogress willreturn }
+; CHECK: attributes #[[ATTR1]] = { mustprogress nosync willreturn memory(read) }
+; CHECK: attributes #[[ATTR2]] = { mustprogress nosync willreturn memory(none) }
+; CHECK: attributes #[[ATTR3]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
+; CHECK: attributes #[[ATTR4]] = { nosync memory(read) }
+; CHECK: attributes #[[ATTR5]] = { nosync memory(none) }
+; CHECK: attributes #[[ATTR6]] = { nofree nosync nounwind memory(argmem: readwrite, inaccessiblemem: readwrite) }
+; CHECK: attributes #[[ATTR7:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+; CHECK: attributes #[[ATTR8:[0-9]+]] = { nofree nosync nounwind memory(inaccessiblemem: read) }
+; CHECK: attributes #[[ATTR9:[0-9]+]] = { nounwind }
+; CHECK: attributes #[[ATTR10]] = { mustprogress nofree norecurse nosync nounwind willreturn }
+; CHECK: attributes #[[ATTR11:[0-9]+]] = { convergent nounwind }
+; CHECK: attributes #[[ATTR12:[0-9]+]] = { nofree nosync nounwind memory(read) }
+; CHECK: attributes #[[ATTR13:[0-9]+]] = { memory(read) }
+; CHECK: attributes #[[ATTR14:[0-9]+]] = { memory(none) }
+; CHECK: attributes #[[ATTR15]] = { willreturn }
+; CHECK: attributes #[[ATTR16]] = { nosync willreturn }
+; CHECK: attributes #[[ATTR17]] = { nosync }
+; CHECK: attributes #[[ATTR18]] = { nofree willreturn }
+; CHECK: attributes #[[ATTR19]] = { nofree nounwind memory(read) }
+; CHECK: attributes #[[ATTR20]] = { nofree willreturn memory(readwrite) }
+;.
+; CHECK: [[META0:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK: [[META1:![0-9]+]] = !{!2}
+; CHECK: [[META2:![0-9]+]] = !{i64 2, i64 -1, i64 -1, i1 true}
+;.
diff --git a/llvm/test/Transforms/OpenMP/parallel_region_merging.ll b/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
index f169fea09d0ea..d31f28382e7ee 100644
--- a/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
+++ b/llvm/test/Transforms/OpenMP/parallel_region_merging.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --include-generated-funcs
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-attributes --check-globals --include-generated-funcs
; RUN: opt -S -aa-pipeline= -passes='attributor,cgscc(openmp-opt-cgscc)' -openmp-opt-enable-merging < %s | FileCheck %s --check-prefix=CHECK2
; #include <omp.h>
; void foo();
@@ -4670,17 +4670,23 @@ entry:
; CHECK-NEXT: call void @use(i32 [[TMP0]])
; CHECK-NEXT: ret void
+;.
+; CHECK2: @[[GLOB0:[0-9]+]] = private unnamed_addr constant [23 x i8] c"
+; CHECK2: @[[GLOB1:[0-9]+]] = private unnamed_addr constant [[STRUCT_IDENT_T:%.*]] { i32 0, i32 2, i32 0, i32 0, ptr @[[GLOB0]] }, align 8
+; CHECK2: @[[GLOB2:[0-9]+]] = private unnamed_addr constant [[STRUCT_IDENT_T:%.*]] { i32 0, i32 2, i32 0, i32 22, ptr @[[GLOB0]] }, align 8
+; CHECK2: @[[GLOB3:[0-9]+]] = private unnamed_addr constant [[STRUCT_IDENT_T:%.*]] { i32 0, i32 66, i32 0, i32 22, ptr @[[GLOB0]] }, align 8
+;.
; CHECK2-LABEL: define {{[^@]+}}@merge
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
; CHECK2-NEXT: [[STRUCTARG:%.*]] = alloca { ptr }, align 8
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr }, ptr [[STRUCTARG]], i32 0, i32 0
-; CHECK2-NEXT: store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr }, ptr [[STRUCTARG]], i32 0, i32 0
+; CHECK2-NEXT: store ptr [[A_ADDR1]], ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge..omp_par, ptr [[STRUCTARG]])
; CHECK2-NEXT: br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
; CHECK2: omp.par.outlined.exit:
@@ -4691,11 +4697,12 @@ entry:
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK2-NEXT: omp.par.entry:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr }, ptr [[TMP0]], i32 0, i32 0
-; CHECK2-NEXT: [[LOADGEP_A_ADDR:%.*]] = load ptr, ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr }, ptr [[TMP0]], i32 0, i32 0
+; CHECK2-NEXT: [[LOADGEP_A_ADDR1:%.*]] = load ptr, ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: [[TID_ADDR_LOCAL:%.*]] = alloca i32, align 4
; CHECK2-NEXT: [[TMP1:%.*]] = load i32, ptr [[TID_ADDR]], align 4
; CHECK2-NEXT: store i32 [[TMP1]], ptr [[TID_ADDR_LOCAL]], align 4
@@ -4704,10 +4711,10 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined.(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined.(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[OMP_GLOBAL_THREAD_NUM]])
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..1(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..1(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: br label [[ENTRY_SPLIT:%.*]]
; CHECK2: entry.split:
; CHECK2-NEXT: br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -4738,12 +4745,12 @@ entry:
; CHECK2-LABEL: define {{[^@]+}}@unmergable_proc_bind
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1:[0-9]+]])
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]])
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
; CHECK2-NEXT: call void @__kmpc_push_proc_bind(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]], i32 noundef 3)
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..2, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..3, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..2, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..3, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: ret void
;
;
@@ -4766,12 +4773,12 @@ entry:
; CHECK2-LABEL: define {{[^@]+}}@unmergable_num_threads
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
; CHECK2-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]])
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
; CHECK2-NEXT: call void @__kmpc_push_num_threads(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]], i32 [[A]])
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..4, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..5, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..4, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..5, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: ret void
;
;
@@ -4794,11 +4801,11 @@ entry:
; CHECK2-LABEL: define {{[^@]+}}@unmergable_seq_call
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..6, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..6, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: call void (...) @foo()
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..7, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..7, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: ret void
;
;
@@ -4822,13 +4829,13 @@ entry:
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
; CHECK2-NEXT: [[STRUCTARG:%.*]] = alloca { ptr }, align 8
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr }, ptr [[STRUCTARG]], i32 0, i32 0
-; CHECK2-NEXT: store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr }, ptr [[STRUCTARG]], i32 0, i32 0
+; CHECK2-NEXT: store ptr [[A_ADDR1]], ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq..omp_par, ptr [[STRUCTARG]])
; CHECK2-NEXT: br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
; CHECK2: omp.par.outlined.exit:
@@ -4836,16 +4843,17 @@ entry:
; CHECK2: omp.par.exit.split:
; CHECK2-NEXT: br label [[ENTRY_SPLIT_SPLIT:%.*]]
; CHECK2: entry.split.split:
-; CHECK2-NEXT: [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
+; CHECK2-NEXT: [[TMP0:%.*]] = load i32, ptr [[A_ADDR1]], align 4
; CHECK2-NEXT: call void @use(i32 [[TMP0]])
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge_seq..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] {
; CHECK2-NEXT: omp.par.entry:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr }, ptr [[TMP0]], i32 0, i32 0
-; CHECK2-NEXT: [[LOADGEP_A_ADDR:%.*]] = load ptr, ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr }, ptr [[TMP0]], i32 0, i32 0
+; CHECK2-NEXT: [[LOADGEP_A_ADDR1:%.*]] = load ptr, ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: [[TID_ADDR_LOCAL:%.*]] = alloca i32, align 4
; CHECK2-NEXT: [[TMP1:%.*]] = load i32, ptr [[TID_ADDR]], align 4
; CHECK2-NEXT: store i32 [[TMP1]], ptr [[TID_ADDR_LOCAL]], align 4
@@ -4854,19 +4862,19 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..8(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..8(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: [[TMP2:%.*]] = call i32 @__kmpc_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
; CHECK2-NEXT: [[TMP3:%.*]] = icmp ne i32 [[TMP2]], 0
; CHECK2-NEXT: br i1 [[TMP3]], label [[OMP_REGION_BODY:%.*]], label [[OMP_REGION_END:%.*]]
; CHECK2: omp_region.end:
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
; CHECK2: omp.par.merged.split.split:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..9(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..9(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: br label [[ENTRY_SPLIT:%.*]]
; CHECK2: entry.split:
; CHECK2-NEXT: br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -4877,9 +4885,9 @@ entry:
; CHECK2: omp_region.body:
; CHECK2-NEXT: br label [[SEQ_PAR_MERGED:%.*]]
; CHECK2: seq.par.merged:
-; CHECK2-NEXT: [[TMP4:%.*]] = load i32, ptr [[LOADGEP_A_ADDR]], align 4
+; CHECK2-NEXT: [[TMP4:%.*]] = load i32, ptr [[LOADGEP_A_ADDR1]], align 4
; CHECK2-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP4]], 1
-; CHECK2-NEXT: store i32 [[ADD]], ptr [[LOADGEP_A_ADDR]], align 4
+; CHECK2-NEXT: store i32 [[ADD]], ptr [[LOADGEP_A_ADDR1]], align 4
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT:%.*]]
; CHECK2: omp.par.merged.split:
; CHECK2-NEXT: br label [[OMP_REGION_BODY_SPLIT:%.*]]
@@ -4911,16 +4919,16 @@ entry:
; CHECK2-NEXT: entry:
; CHECK2-NEXT: [[STRUCTARG:%.*]] = alloca { ptr, ptr, ptr }, align 8
; CHECK2-NEXT: [[F_RELOADED:%.*]] = alloca float, align 4
-; CHECK2-NEXT: [[F_ADDR:%.*]] = alloca float, align 4
-; CHECK2-NEXT: store float [[F]], ptr [[F_ADDR]], align 4
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: [[F_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store float [[F]], ptr [[F_ADDR1]], align 4
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: store float [[F]], ptr [[F_RELOADED]], align 4
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
; CHECK2-NEXT: [[GEP_F_RELOADED:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 0
; CHECK2-NEXT: store ptr [[F_RELOADED]], ptr [[GEP_F_RELOADED]], align 8
-; CHECK2-NEXT: [[GEP_F_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
-; CHECK2-NEXT: store ptr [[F_ADDR]], ptr [[GEP_F_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_F_ADDR1:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
+; CHECK2-NEXT: store ptr [[F_ADDR1]], ptr [[GEP_F_ADDR1]], align 8
; CHECK2-NEXT: [[GEP_P:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
; CHECK2-NEXT: store ptr [[P]], ptr [[GEP_P]], align 8
; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq_float..omp_par, ptr [[STRUCTARG]])
@@ -4933,13 +4941,14 @@ entry:
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge_seq_float..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] {
; CHECK2-NEXT: omp.par.entry:
; CHECK2-NEXT: [[GEP_F_RELOADED:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 0
; CHECK2-NEXT: [[LOADGEP_F_RELOADED:%.*]] = load ptr, ptr [[GEP_F_RELOADED]], align 8
-; CHECK2-NEXT: [[GEP_F_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
-; CHECK2-NEXT: [[LOADGEP_F_ADDR:%.*]] = load ptr, ptr [[GEP_F_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_F_ADDR1:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
+; CHECK2-NEXT: [[LOADGEP_F_ADDR1:%.*]] = load ptr, ptr [[GEP_F_ADDR1]], align 8
; CHECK2-NEXT: [[GEP_P:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 2
; CHECK2-NEXT: [[LOADGEP_P:%.*]] = load ptr, ptr [[GEP_P]], align 8
; CHECK2-NEXT: [[TID_ADDR_LOCAL:%.*]] = alloca i32, align 4
@@ -4951,19 +4960,19 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..10(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_F_ADDR]])
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..10(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_F_ADDR1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: [[TMP3:%.*]] = call i32 @__kmpc_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
; CHECK2-NEXT: [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
; CHECK2-NEXT: br i1 [[TMP4]], label [[OMP_REGION_BODY:%.*]], label [[OMP_REGION_END:%.*]]
; CHECK2: omp_region.end:
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
; CHECK2: omp.par.merged.split.split:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..11(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_F_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..11(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_F_ADDR1]])
; CHECK2-NEXT: br label [[ENTRY_SPLIT:%.*]]
; CHECK2: entry.split:
; CHECK2-NEXT: br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5009,13 +5018,13 @@ entry:
; CHECK2-NEXT: entry:
; CHECK2-NEXT: [[STRUCTARG:%.*]] = alloca { ptr, ptr }, align 8
; CHECK2-NEXT: [[A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC:%.*]] = alloca i64, align 8
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 0
-; CHECK2-NEXT: store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 0
+; CHECK2-NEXT: store ptr [[A_ADDR1]], ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: [[GEP_A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC:%.*]] = getelementptr { ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
; CHECK2-NEXT: store ptr [[A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC]], ptr [[GEP_A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC]], align 8
; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq_firstprivate..omp_par, ptr [[STRUCTARG]])
@@ -5025,16 +5034,17 @@ entry:
; CHECK2: omp.par.exit.split:
; CHECK2-NEXT: br label [[ENTRY_SPLIT_SPLIT:%.*]]
; CHECK2: entry.split.split:
-; CHECK2-NEXT: [[TMP0:%.*]] = load i32, ptr [[A_ADDR]], align 4
+; CHECK2-NEXT: [[TMP0:%.*]] = load i32, ptr [[A_ADDR1]], align 4
; CHECK2-NEXT: call void @use(i32 [[TMP0]])
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge_seq_firstprivate..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] {
; CHECK2-NEXT: omp.par.entry:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr, ptr }, ptr [[TMP0]], i32 0, i32 0
-; CHECK2-NEXT: [[LOADGEP_A_ADDR:%.*]] = load ptr, ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr, ptr }, ptr [[TMP0]], i32 0, i32 0
+; CHECK2-NEXT: [[LOADGEP_A_ADDR1:%.*]] = load ptr, ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: [[GEP_A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC:%.*]] = getelementptr { ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
; CHECK2-NEXT: [[LOADGEP_A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC:%.*]] = load ptr, ptr [[GEP_A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC]], align 8
; CHECK2-NEXT: [[TID_ADDR_LOCAL:%.*]] = alloca i32, align 4
@@ -5045,16 +5055,16 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..12(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..12(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: [[TMP2:%.*]] = call i32 @__kmpc_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
; CHECK2-NEXT: [[TMP3:%.*]] = icmp ne i32 [[TMP2]], 0
; CHECK2-NEXT: br i1 [[TMP3]], label [[OMP_REGION_BODY:%.*]], label [[OMP_REGION_END:%.*]]
; CHECK2: omp_region.end:
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
; CHECK2: omp.par.merged.split.split:
; CHECK2-NEXT: [[A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_LOAD:%.*]] = load i64, ptr [[LOADGEP_A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC]], align 8
@@ -5069,9 +5079,9 @@ entry:
; CHECK2: omp_region.body:
; CHECK2-NEXT: br label [[SEQ_PAR_MERGED:%.*]]
; CHECK2: seq.par.merged:
-; CHECK2-NEXT: [[TMP4:%.*]] = load i32, ptr [[LOADGEP_A_ADDR]], align 4
+; CHECK2-NEXT: [[TMP4:%.*]] = load i32, ptr [[LOADGEP_A_ADDR1]], align 4
; CHECK2-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP4]], 1
-; CHECK2-NEXT: store i32 [[ADD]], ptr [[LOADGEP_A_ADDR]], align 4
+; CHECK2-NEXT: store i32 [[ADD]], ptr [[LOADGEP_A_ADDR1]], align 4
; CHECK2-NEXT: [[A_CASTED_SROA_0_0_INSERT_EXT:%.*]] = zext i32 [[ADD]] to i64
; CHECK2-NEXT: store i64 [[A_CASTED_SROA_0_0_INSERT_EXT]], ptr [[LOADGEP_A_CASTED_SROA_0_0_INSERT_EXT_SEQ_OUTPUT_ALLOC]], align 8
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT:%.*]]
@@ -5104,13 +5114,13 @@ entry:
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
; CHECK2-NEXT: [[STRUCTARG:%.*]] = alloca { ptr }, align 8
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr }, ptr [[STRUCTARG]], i32 0, i32 0
-; CHECK2-NEXT: store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr }, ptr [[STRUCTARG]], i32 0, i32 0
+; CHECK2-NEXT: store ptr [[A_ADDR1]], ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq_sink_lt..omp_par, ptr [[STRUCTARG]])
; CHECK2-NEXT: br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
; CHECK2: omp.par.outlined.exit:
@@ -5121,12 +5131,13 @@ entry:
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge_seq_sink_lt..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] {
; CHECK2-NEXT: omp.par.entry:
; CHECK2-NEXT: [[B:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr }, ptr [[TMP0]], i32 0, i32 0
-; CHECK2-NEXT: [[LOADGEP_A_ADDR:%.*]] = load ptr, ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr }, ptr [[TMP0]], i32 0, i32 0
+; CHECK2-NEXT: [[LOADGEP_A_ADDR1:%.*]] = load ptr, ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: [[TID_ADDR_LOCAL:%.*]] = alloca i32, align 4
; CHECK2-NEXT: [[TMP1:%.*]] = load i32, ptr [[TID_ADDR]], align 4
; CHECK2-NEXT: store i32 [[TMP1]], ptr [[TID_ADDR_LOCAL]], align 4
@@ -5135,19 +5146,19 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..14(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..14(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: [[TMP2:%.*]] = call i32 @__kmpc_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
; CHECK2-NEXT: [[TMP3:%.*]] = icmp ne i32 [[TMP2]], 0
; CHECK2-NEXT: br i1 [[TMP3]], label [[OMP_REGION_BODY:%.*]], label [[OMP_REGION_END:%.*]]
; CHECK2: omp_region.end:
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
; CHECK2: omp.par.merged.split.split:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..15(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..15(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: br label [[ENTRY_SPLIT:%.*]]
; CHECK2: entry.split:
; CHECK2-NEXT: br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5194,20 +5205,20 @@ entry:
; CHECK2-NEXT: entry:
; CHECK2-NEXT: [[STRUCTARG:%.*]] = alloca { ptr, ptr, ptr }, align 8
; CHECK2-NEXT: [[A_RELOADED:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: [[B:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: [[B2:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM4:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: store i32 [[A]], ptr [[A_RELOADED]], align 4
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
; CHECK2-NEXT: [[GEP_A_RELOADED:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 0
; CHECK2-NEXT: store ptr [[A_RELOADED]], ptr [[GEP_A_RELOADED]], align 8
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
-; CHECK2-NEXT: store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
-; CHECK2-NEXT: [[GEP_B:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
-; CHECK2-NEXT: store ptr [[B]], ptr [[GEP_B]], align 8
-; CHECK2-NEXT: call void @llvm.lifetime.start.p0(i64 -1, ptr [[B]])
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
+; CHECK2-NEXT: store ptr [[A_ADDR1]], ptr [[GEP_A_ADDR1]], align 8
+; CHECK2-NEXT: [[GEP_B2:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
+; CHECK2-NEXT: store ptr [[B2]], ptr [[GEP_B2]], align 8
+; CHECK2-NEXT: call void @llvm.lifetime.start.p0(i64 -1, ptr [[B2]])
; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_seq_par_use..omp_par, ptr [[STRUCTARG]])
; CHECK2-NEXT: br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
; CHECK2: omp.par.outlined.exit:
@@ -5215,19 +5226,20 @@ entry:
; CHECK2: omp.par.exit.split:
; CHECK2-NEXT: br label [[ENTRY_SPLIT_SPLIT:%.*]]
; CHECK2: entry.split.split:
-; CHECK2-NEXT: call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B]])
+; CHECK2-NEXT: call void @llvm.lifetime.end.p0(i64 noundef 4, ptr noundef nonnull [[B2]])
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge_seq_par_use..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] {
; CHECK2-NEXT: omp.par.entry:
; CHECK2-NEXT: [[GEP_A_RELOADED:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 0
; CHECK2-NEXT: [[LOADGEP_A_RELOADED:%.*]] = load ptr, ptr [[GEP_A_RELOADED]], align 8
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
-; CHECK2-NEXT: [[LOADGEP_A_ADDR:%.*]] = load ptr, ptr [[GEP_A_ADDR]], align 8
-; CHECK2-NEXT: [[GEP_B:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 2
-; CHECK2-NEXT: [[LOADGEP_B:%.*]] = load ptr, ptr [[GEP_B]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
+; CHECK2-NEXT: [[LOADGEP_A_ADDR1:%.*]] = load ptr, ptr [[GEP_A_ADDR1]], align 8
+; CHECK2-NEXT: [[GEP_B2:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 2
+; CHECK2-NEXT: [[LOADGEP_B2:%.*]] = load ptr, ptr [[GEP_B2]], align 8
; CHECK2-NEXT: [[TID_ADDR_LOCAL:%.*]] = alloca i32, align 4
; CHECK2-NEXT: [[TMP1:%.*]] = load i32, ptr [[TID_ADDR]], align 4
; CHECK2-NEXT: store i32 [[TMP1]], ptr [[TID_ADDR_LOCAL]], align 4
@@ -5237,19 +5249,19 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..16(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..16(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: [[TMP3:%.*]] = call i32 @__kmpc_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
; CHECK2-NEXT: [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
; CHECK2-NEXT: br i1 [[TMP4]], label [[OMP_REGION_BODY:%.*]], label [[OMP_REGION_END:%.*]]
; CHECK2: omp_region.end:
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM3]])
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
; CHECK2: omp.par.merged.split.split:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..17(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_B]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..17(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_B2]])
; CHECK2-NEXT: br label [[ENTRY_SPLIT:%.*]]
; CHECK2: entry.split:
; CHECK2-NEXT: br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5261,7 +5273,7 @@ entry:
; CHECK2-NEXT: br label [[SEQ_PAR_MERGED:%.*]]
; CHECK2: seq.par.merged:
; CHECK2-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP2]], 1
-; CHECK2-NEXT: store i32 [[ADD]], ptr [[LOADGEP_B]], align 4
+; CHECK2-NEXT: store i32 [[ADD]], ptr [[LOADGEP_B2]], align 4
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT:%.*]]
; CHECK2: omp.par.merged.split:
; CHECK2-NEXT: br label [[OMP_REGION_BODY_SPLIT:%.*]]
@@ -5294,17 +5306,17 @@ entry:
; CHECK2-SAME: (i32 [[CANCEL1:%.*]], i32 [[CANCEL2:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
; CHECK2-NEXT: [[STRUCTARG:%.*]] = alloca { ptr, ptr }, align 8
-; CHECK2-NEXT: [[CANCEL1_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: [[CANCEL2_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[CANCEL1]], ptr [[CANCEL1_ADDR]], align 4
-; CHECK2-NEXT: store i32 [[CANCEL2]], ptr [[CANCEL2_ADDR]], align 4
+; CHECK2-NEXT: [[CANCEL1_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: [[CANCEL2_ADDR2:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[CANCEL1]], ptr [[CANCEL1_ADDR1]], align 4
+; CHECK2-NEXT: store i32 [[CANCEL2]], ptr [[CANCEL2_ADDR2]], align 4
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
-; CHECK2-NEXT: [[GEP_CANCEL1_ADDR:%.*]] = getelementptr { ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 0
-; CHECK2-NEXT: store ptr [[CANCEL1_ADDR]], ptr [[GEP_CANCEL1_ADDR]], align 8
-; CHECK2-NEXT: [[GEP_CANCEL2_ADDR:%.*]] = getelementptr { ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
-; CHECK2-NEXT: store ptr [[CANCEL2_ADDR]], ptr [[GEP_CANCEL2_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_CANCEL1_ADDR1:%.*]] = getelementptr { ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 0
+; CHECK2-NEXT: store ptr [[CANCEL1_ADDR1]], ptr [[GEP_CANCEL1_ADDR1]], align 8
+; CHECK2-NEXT: [[GEP_CANCEL2_ADDR2:%.*]] = getelementptr { ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
+; CHECK2-NEXT: store ptr [[CANCEL2_ADDR2]], ptr [[GEP_CANCEL2_ADDR2]], align 8
; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_cancellable_regions..omp_par, ptr [[STRUCTARG]])
; CHECK2-NEXT: br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
; CHECK2: omp.par.outlined.exit:
@@ -5315,13 +5327,14 @@ entry:
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge_cancellable_regions..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] {
; CHECK2-NEXT: omp.par.entry:
-; CHECK2-NEXT: [[GEP_CANCEL1_ADDR:%.*]] = getelementptr { ptr, ptr }, ptr [[TMP0]], i32 0, i32 0
-; CHECK2-NEXT: [[LOADGEP_CANCEL1_ADDR:%.*]] = load ptr, ptr [[GEP_CANCEL1_ADDR]], align 8
-; CHECK2-NEXT: [[GEP_CANCEL2_ADDR:%.*]] = getelementptr { ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
-; CHECK2-NEXT: [[LOADGEP_CANCEL2_ADDR:%.*]] = load ptr, ptr [[GEP_CANCEL2_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_CANCEL1_ADDR1:%.*]] = getelementptr { ptr, ptr }, ptr [[TMP0]], i32 0, i32 0
+; CHECK2-NEXT: [[LOADGEP_CANCEL1_ADDR1:%.*]] = load ptr, ptr [[GEP_CANCEL1_ADDR1]], align 8
+; CHECK2-NEXT: [[GEP_CANCEL2_ADDR2:%.*]] = getelementptr { ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
+; CHECK2-NEXT: [[LOADGEP_CANCEL2_ADDR2:%.*]] = load ptr, ptr [[GEP_CANCEL2_ADDR2]], align 8
; CHECK2-NEXT: [[TID_ADDR_LOCAL:%.*]] = alloca i32, align 4
; CHECK2-NEXT: [[TMP1:%.*]] = load i32, ptr [[TID_ADDR]], align 4
; CHECK2-NEXT: store i32 [[TMP1]], ptr [[TID_ADDR_LOCAL]], align 4
@@ -5330,10 +5343,10 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..18(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL1_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..18(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL1_ADDR1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM]])
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..19(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL2_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..19(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL2_ADDR2]])
; CHECK2-NEXT: br label [[ENTRY_SPLIT:%.*]]
; CHECK2: entry.split:
; CHECK2-NEXT: br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5378,20 +5391,20 @@ entry:
; CHECK2-NEXT: entry:
; CHECK2-NEXT: [[STRUCTARG:%.*]] = alloca { ptr, ptr, ptr }, align 8
; CHECK2-NEXT: [[CANCEL1_RELOADED:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: [[CANCEL1_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: [[CANCEL2_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[CANCEL1]], ptr [[CANCEL1_ADDR]], align 4
-; CHECK2-NEXT: store i32 [[CANCEL2]], ptr [[CANCEL2_ADDR]], align 4
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: [[CANCEL1_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: [[CANCEL2_ADDR2:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[CANCEL1]], ptr [[CANCEL1_ADDR1]], align 4
+; CHECK2-NEXT: store i32 [[CANCEL2]], ptr [[CANCEL2_ADDR2]], align 4
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM4:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: store i32 [[CANCEL1]], ptr [[CANCEL1_RELOADED]], align 4
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
; CHECK2-NEXT: [[GEP_CANCEL1_RELOADED:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 0
; CHECK2-NEXT: store ptr [[CANCEL1_RELOADED]], ptr [[GEP_CANCEL1_RELOADED]], align 8
-; CHECK2-NEXT: [[GEP_CANCEL1_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
-; CHECK2-NEXT: store ptr [[CANCEL1_ADDR]], ptr [[GEP_CANCEL1_ADDR]], align 8
-; CHECK2-NEXT: [[GEP_CANCEL2_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
-; CHECK2-NEXT: store ptr [[CANCEL2_ADDR]], ptr [[GEP_CANCEL2_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_CANCEL1_ADDR1:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
+; CHECK2-NEXT: store ptr [[CANCEL1_ADDR1]], ptr [[GEP_CANCEL1_ADDR1]], align 8
+; CHECK2-NEXT: [[GEP_CANCEL2_ADDR2:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
+; CHECK2-NEXT: store ptr [[CANCEL2_ADDR2]], ptr [[GEP_CANCEL2_ADDR2]], align 8
; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_cancellable_regions_seq..omp_par, ptr [[STRUCTARG]])
; CHECK2-NEXT: br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
; CHECK2: omp.par.outlined.exit:
@@ -5402,15 +5415,16 @@ entry:
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge_cancellable_regions_seq..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] {
; CHECK2-NEXT: omp.par.entry:
; CHECK2-NEXT: [[GEP_CANCEL1_RELOADED:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 0
; CHECK2-NEXT: [[LOADGEP_CANCEL1_RELOADED:%.*]] = load ptr, ptr [[GEP_CANCEL1_RELOADED]], align 8
-; CHECK2-NEXT: [[GEP_CANCEL1_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
-; CHECK2-NEXT: [[LOADGEP_CANCEL1_ADDR:%.*]] = load ptr, ptr [[GEP_CANCEL1_ADDR]], align 8
-; CHECK2-NEXT: [[GEP_CANCEL2_ADDR:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 2
-; CHECK2-NEXT: [[LOADGEP_CANCEL2_ADDR:%.*]] = load ptr, ptr [[GEP_CANCEL2_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_CANCEL1_ADDR1:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
+; CHECK2-NEXT: [[LOADGEP_CANCEL1_ADDR1:%.*]] = load ptr, ptr [[GEP_CANCEL1_ADDR1]], align 8
+; CHECK2-NEXT: [[GEP_CANCEL2_ADDR2:%.*]] = getelementptr { ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 2
+; CHECK2-NEXT: [[LOADGEP_CANCEL2_ADDR2:%.*]] = load ptr, ptr [[GEP_CANCEL2_ADDR2]], align 8
; CHECK2-NEXT: [[TID_ADDR_LOCAL:%.*]] = alloca i32, align 4
; CHECK2-NEXT: [[TMP1:%.*]] = load i32, ptr [[TID_ADDR]], align 4
; CHECK2-NEXT: store i32 [[TMP1]], ptr [[TID_ADDR_LOCAL]], align 4
@@ -5420,19 +5434,19 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..20(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL1_ADDR]])
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..20(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL1_ADDR1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: [[TMP3:%.*]] = call i32 @__kmpc_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
; CHECK2-NEXT: [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
; CHECK2-NEXT: br i1 [[TMP4]], label [[OMP_REGION_BODY:%.*]], label [[OMP_REGION_END:%.*]]
; CHECK2: omp_region.end:
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM3]])
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
; CHECK2: omp.par.merged.split.split:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..21(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL2_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..21(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_CANCEL2_ADDR2]])
; CHECK2-NEXT: br label [[ENTRY_SPLIT:%.*]]
; CHECK2: entry.split:
; CHECK2-NEXT: br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5445,7 +5459,7 @@ entry:
; CHECK2: seq.par.merged:
; CHECK2-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP2]], 0
; CHECK2-NEXT: [[LNOT_EXT:%.*]] = zext i1 [[TOBOOL_NOT]] to i32
-; CHECK2-NEXT: store i32 [[LNOT_EXT]], ptr [[LOADGEP_CANCEL2_ADDR]], align 4
+; CHECK2-NEXT: store i32 [[LNOT_EXT]], ptr [[LOADGEP_CANCEL2_ADDR2]], align 4
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT:%.*]]
; CHECK2: omp.par.merged.split:
; CHECK2-NEXT: br label [[OMP_REGION_BODY_SPLIT:%.*]]
@@ -5488,13 +5502,13 @@ entry:
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
; CHECK2-NEXT: [[STRUCTARG:%.*]] = alloca { ptr }, align 8
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr }, ptr [[STRUCTARG]], i32 0, i32 0
-; CHECK2-NEXT: store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr }, ptr [[STRUCTARG]], i32 0, i32 0
+; CHECK2-NEXT: store ptr [[A_ADDR1]], ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_3..omp_par, ptr [[STRUCTARG]])
; CHECK2-NEXT: br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
; CHECK2: omp.par.outlined.exit:
@@ -5505,11 +5519,12 @@ entry:
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge_3..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] {
; CHECK2-NEXT: omp.par.entry:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr }, ptr [[TMP0]], i32 0, i32 0
-; CHECK2-NEXT: [[LOADGEP_A_ADDR:%.*]] = load ptr, ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr }, ptr [[TMP0]], i32 0, i32 0
+; CHECK2-NEXT: [[LOADGEP_A_ADDR1:%.*]] = load ptr, ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: [[TID_ADDR_LOCAL:%.*]] = alloca i32, align 4
; CHECK2-NEXT: [[TMP1:%.*]] = load i32, ptr [[TID_ADDR]], align 4
; CHECK2-NEXT: store i32 [[TMP1]], ptr [[TID_ADDR_LOCAL]], align 4
@@ -5518,13 +5533,13 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..22(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..22(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM]])
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..23(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..23(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..24(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..24(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: br label [[ENTRY_SPLIT:%.*]]
; CHECK2: entry.split:
; CHECK2-NEXT: br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5567,16 +5582,16 @@ entry:
; CHECK2-NEXT: [[A_RELOADED:%.*]] = alloca i32, align 4
; CHECK2-NEXT: [[ADD1_SEQ_OUTPUT_ALLOC:%.*]] = alloca i32, align 4
; CHECK2-NEXT: [[ADD_SEQ_OUTPUT_ALLOC:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM7:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM8:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: store i32 [[A]], ptr [[A_RELOADED]], align 4
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
; CHECK2-NEXT: [[GEP_A_RELOADED:%.*]] = getelementptr { ptr, ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 0
; CHECK2-NEXT: store ptr [[A_RELOADED]], ptr [[GEP_A_RELOADED]], align 8
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr, ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
-; CHECK2-NEXT: store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr, ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 1
+; CHECK2-NEXT: store ptr [[A_ADDR1]], ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: [[GEP_ADD_SEQ_OUTPUT_ALLOC:%.*]] = getelementptr { ptr, ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 2
; CHECK2-NEXT: store ptr [[ADD_SEQ_OUTPUT_ALLOC]], ptr [[GEP_ADD_SEQ_OUTPUT_ALLOC]], align 8
; CHECK2-NEXT: [[GEP_ADD1_SEQ_OUTPUT_ALLOC:%.*]] = getelementptr { ptr, ptr, ptr, ptr }, ptr [[STRUCTARG]], i32 0, i32 3
@@ -5593,13 +5608,14 @@ entry:
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge_3_seq..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] {
; CHECK2-NEXT: omp.par.entry:
; CHECK2-NEXT: [[GEP_A_RELOADED:%.*]] = getelementptr { ptr, ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 0
; CHECK2-NEXT: [[LOADGEP_A_RELOADED:%.*]] = load ptr, ptr [[GEP_A_RELOADED]], align 8
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr, ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
-; CHECK2-NEXT: [[LOADGEP_A_ADDR:%.*]] = load ptr, ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr, ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 1
+; CHECK2-NEXT: [[LOADGEP_A_ADDR1:%.*]] = load ptr, ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: [[GEP_ADD_SEQ_OUTPUT_ALLOC:%.*]] = getelementptr { ptr, ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 2
; CHECK2-NEXT: [[LOADGEP_ADD_SEQ_OUTPUT_ALLOC:%.*]] = load ptr, ptr [[GEP_ADD_SEQ_OUTPUT_ALLOC]], align 8
; CHECK2-NEXT: [[GEP_ADD1_SEQ_OUTPUT_ALLOC:%.*]] = getelementptr { ptr, ptr, ptr, ptr }, ptr [[TMP0]], i32 0, i32 3
@@ -5613,31 +5629,31 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..25(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..25(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: [[TMP3:%.*]] = call i32 @__kmpc_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM]])
; CHECK2-NEXT: [[TMP4:%.*]] = icmp ne i32 [[TMP3]], 0
; CHECK2-NEXT: br i1 [[TMP4]], label [[OMP_REGION_BODY:%.*]], label [[OMP_REGION_END:%.*]]
; CHECK2: omp_region.end:
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM1:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM1]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM2:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM2]])
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT_SPLIT:%.*]]
; CHECK2: omp.par.merged.split.split:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..26(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM4:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM4]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..26(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM3:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: [[TMP5:%.*]] = call i32 @__kmpc_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM3]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM3]])
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM4:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: [[TMP5:%.*]] = call i32 @__kmpc_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM4]])
; CHECK2-NEXT: [[TMP6:%.*]] = icmp ne i32 [[TMP5]], 0
-; CHECK2-NEXT: br i1 [[TMP6]], label [[OMP_REGION_BODY5:%.*]], label [[OMP_REGION_END4:%.*]]
-; CHECK2: omp_region.end4:
-; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM6:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
-; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM6]])
+; CHECK2-NEXT: br i1 [[TMP6]], label [[OMP_REGION_BODY6:%.*]], label [[OMP_REGION_END5:%.*]]
+; CHECK2: omp_region.end5:
+; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM7:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
+; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM7]])
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT_SPLIT_SPLIT_SPLIT:%.*]]
; CHECK2: omp.par.merged.split.split.split.split:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..27(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..27(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: br label [[ENTRY_SPLIT:%.*]]
; CHECK2: entry.split:
; CHECK2-NEXT: br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5645,18 +5661,18 @@ entry:
; CHECK2-NEXT: br label [[OMP_PAR_PRE_FINALIZE:%.*]]
; CHECK2: omp.par.pre_finalize:
; CHECK2-NEXT: br label [[OMP_PAR_OUTLINED_EXIT_EXITSTUB:%.*]]
-; CHECK2: omp_region.body5:
-; CHECK2-NEXT: br label [[SEQ_PAR_MERGED2:%.*]]
-; CHECK2: seq.par.merged2:
+; CHECK2: omp_region.body6:
+; CHECK2-NEXT: br label [[SEQ_PAR_MERGED3:%.*]]
+; CHECK2: seq.par.merged3:
; CHECK2-NEXT: [[ADD_SEQ_OUTPUT_LOAD:%.*]] = load i32, ptr [[LOADGEP_ADD_SEQ_OUTPUT_ALLOC]], align 4
; CHECK2-NEXT: [[ADD1:%.*]] = add nsw i32 [[ADD_SEQ_OUTPUT_LOAD]], [[TMP2]]
; CHECK2-NEXT: store i32 [[ADD1]], ptr [[LOADGEP_ADD1_SEQ_OUTPUT_ALLOC]], align 4
; CHECK2-NEXT: br label [[OMP_PAR_MERGED_SPLIT_SPLIT_SPLIT:%.*]]
; CHECK2: omp.par.merged.split.split.split:
-; CHECK2-NEXT: br label [[OMP_REGION_BODY5_SPLIT:%.*]]
-; CHECK2: omp_region.body5.split:
-; CHECK2-NEXT: call void @__kmpc_end_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM3]])
-; CHECK2-NEXT: br label [[OMP_REGION_END4]]
+; CHECK2-NEXT: br label [[OMP_REGION_BODY6_SPLIT:%.*]]
+; CHECK2: omp_region.body6.split:
+; CHECK2-NEXT: call void @__kmpc_end_master(ptr @[[GLOB2]], i32 [[OMP_GLOBAL_THREAD_NUM4]])
+; CHECK2-NEXT: br label [[OMP_REGION_END5]]
; CHECK2: omp_region.body:
; CHECK2-NEXT: br label [[SEQ_PAR_MERGED:%.*]]
; CHECK2: seq.par.merged:
@@ -5699,13 +5715,13 @@ entry:
; CHECK2-LABEL: define {{[^@]+}}@unmergable_3_seq_call
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..28, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..28, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: call void (...) @foo()
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..29, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..29, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: call void (...) @foo()
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..30, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..30, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: ret void
;
;
@@ -5736,13 +5752,13 @@ entry:
; CHECK2-LABEL: define {{[^@]+}}@unmergable_3_proc_bind
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
; CHECK2-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]])
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..31, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..31, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: call void @__kmpc_push_proc_bind(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]], i32 noundef 3)
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..32, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..33, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..32, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..33, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: ret void
;
;
@@ -5773,13 +5789,13 @@ entry:
; CHECK2-LABEL: define {{[^@]+}}@unmergable_3_num_threads
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
; CHECK2-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]])
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..34, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..34, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: call void @__kmpc_push_num_threads(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 [[TMP0]], i32 [[A]])
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..35, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..36, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..35, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..36, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: ret void
;
;
@@ -5811,13 +5827,13 @@ entry:
; CHECK2-SAME: (i32 [[A:%.*]]) local_unnamed_addr {
; CHECK2-NEXT: entry:
; CHECK2-NEXT: [[STRUCTARG:%.*]] = alloca { ptr }, align 8
-; CHECK2-NEXT: [[A_ADDR:%.*]] = alloca i32, align 4
-; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR]], align 4
+; CHECK2-NEXT: [[A_ADDR1:%.*]] = alloca i8, i32 4, align 4
+; CHECK2-NEXT: store i32 [[A]], ptr [[A_ADDR1]], align 4
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: br label [[OMP_PARALLEL:%.*]]
; CHECK2: omp_parallel:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr }, ptr [[STRUCTARG]], i32 0, i32 0
-; CHECK2-NEXT: store ptr [[A_ADDR]], ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr }, ptr [[STRUCTARG]], i32 0, i32 0
+; CHECK2-NEXT: store ptr [[A_ADDR1]], ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @[[GLOB2]], i32 1, ptr @merge_2_unmergable_1..omp_par, ptr [[STRUCTARG]])
; CHECK2-NEXT: br label [[OMP_PAR_OUTLINED_EXIT:%.*]]
; CHECK2: omp.par.outlined.exit:
@@ -5826,15 +5842,16 @@ entry:
; CHECK2-NEXT: br label [[ENTRY_SPLIT_SPLIT:%.*]]
; CHECK2: entry.split.split:
; CHECK2-NEXT: call void (...) @foo()
-; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..39, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR]])
+; CHECK2-NEXT: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr noundef nonnull align 8 dereferenceable(24) @[[GLOB1]], i32 noundef 1, ptr noundef nonnull @.omp_outlined..39, ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[A_ADDR1]])
; CHECK2-NEXT: ret void
;
;
+; CHECK2: Function Attrs: norecurse nounwind
; CHECK2-LABEL: define {{[^@]+}}@merge_2_unmergable_1..omp_par
; CHECK2-SAME: (ptr noalias [[TID_ADDR:%.*]], ptr noalias [[ZERO_ADDR:%.*]], ptr [[TMP0:%.*]]) #[[ATTR0]] {
; CHECK2-NEXT: omp.par.entry:
-; CHECK2-NEXT: [[GEP_A_ADDR:%.*]] = getelementptr { ptr }, ptr [[TMP0]], i32 0, i32 0
-; CHECK2-NEXT: [[LOADGEP_A_ADDR:%.*]] = load ptr, ptr [[GEP_A_ADDR]], align 8
+; CHECK2-NEXT: [[GEP_A_ADDR1:%.*]] = getelementptr { ptr }, ptr [[TMP0]], i32 0, i32 0
+; CHECK2-NEXT: [[LOADGEP_A_ADDR1:%.*]] = load ptr, ptr [[GEP_A_ADDR1]], align 8
; CHECK2-NEXT: [[TID_ADDR_LOCAL:%.*]] = alloca i32, align 4
; CHECK2-NEXT: [[TMP1:%.*]] = load i32, ptr [[TID_ADDR]], align 4
; CHECK2-NEXT: store i32 [[TMP1]], ptr [[TID_ADDR_LOCAL]], align 4
@@ -5843,10 +5860,10 @@ entry:
; CHECK2: omp.par.region:
; CHECK2-NEXT: br label [[OMP_PAR_MERGED:%.*]]
; CHECK2: omp.par.merged:
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..37(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..37(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: [[OMP_GLOBAL_THREAD_NUM:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
; CHECK2-NEXT: call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[OMP_GLOBAL_THREAD_NUM]])
-; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..38(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR]])
+; CHECK2-NEXT: call void (ptr, ptr, ...) @.omp_outlined..38(ptr [[TID_ADDR]], ptr [[ZERO_ADDR]], ptr nocapture nofree noundef nonnull readonly align 4 dereferenceable(4) [[LOADGEP_A_ADDR1]])
; CHECK2-NEXT: br label [[ENTRY_SPLIT:%.*]]
; CHECK2: entry.split:
; CHECK2-NEXT: br label [[OMP_PAR_REGION_SPLIT:%.*]]
@@ -5881,3 +5898,14 @@ entry:
; CHECK2-NEXT: call void @use(i32 [[TMP0]])
; CHECK2-NEXT: ret void
;
+;.
+; CHECK2: attributes #[[ATTR0]] = { norecurse nounwind }
+; CHECK2: attributes #[[ATTR1:[0-9]+]] = { nounwind }
+; CHECK2: attributes #[[ATTR2:[0-9]+]] = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) }
+; CHECK2: attributes #[[ATTR3:[0-9]+]] = { convergent nounwind }
+;.
+; CHECK2: [[META0:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
+; CHECK2: [[META1:![0-9]+]] = !{i32 7, !"openmp", i32 50}
+; CHECK2: [[META2:![0-9]+]] = !{!3}
+; CHECK2: [[META3:![0-9]+]] = !{i64 2, i64 -1, i64 -1, i1 true}
+;.
>From 1133675c5decfb67a66aa01a784773af8959dcfc Mon Sep 17 00:00:00 2001
From: Vidush Singhal <singhal2 at quartz770.llnl.gov>
Date: Thu, 23 May 2024 15:32:52 -0700
Subject: [PATCH 2/3] use ptrtoint + sub + inttoptr
---
llvm/include/llvm/Transforms/IPO/Attributor.h | 6483 --------
.../Transforms/IPO/AttributorAttributes.cpp | 13216 ----------------
llvm/test/Transforms/Attributor/allocator.ll | 119 +-
3 files changed, 67 insertions(+), 19751 deletions(-)
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index fb96e52121f82..e69de29bb2d1d 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -1,6483 +0,0 @@
-//===- Attributor.h --- Module-wide attribute deduction ---------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Attributor: An inter procedural (abstract) "attribute" deduction framework.
-//
-// The Attributor framework is an inter procedural abstract analysis (fixpoint
-// iteration analysis). The goal is to allow easy deduction of new attributes as
-// well as information exchange between abstract attributes in-flight.
-//
-// The Attributor class is the driver and the link between the various abstract
-// attributes. The Attributor will iterate until a fixpoint state is reached by
-// all abstract attributes in-flight, or until it will enforce a pessimistic fix
-// point because an iteration limit is reached.
-//
-// Abstract attributes, derived from the AbstractAttribute class, actually
-// describe properties of the code. They can correspond to actual LLVM-IR
-// attributes, or they can be more general, ultimately unrelated to LLVM-IR
-// attributes. The latter is useful when an abstract attributes provides
-// information to other abstract attributes in-flight but we might not want to
-// manifest the information. The Attributor allows to query in-flight abstract
-// attributes through the `Attributor::getAAFor` method (see the method
-// description for an example). If the method is used by an abstract attribute
-// P, and it results in an abstract attribute Q, the Attributor will
-// automatically capture a potential dependence from Q to P. This dependence
-// will cause P to be reevaluated whenever Q changes in the future.
-//
-// The Attributor will only reevaluate abstract attributes that might have
-// changed since the last iteration. That means that the Attribute will not
-// revisit all instructions/blocks/functions in the module but only query
-// an update from a subset of the abstract attributes.
-//
-// The update method `AbstractAttribute::updateImpl` is implemented by the
-// specific "abstract attribute" subclasses. The method is invoked whenever the
-// currently assumed state (see the AbstractState class) might not be valid
-// anymore. This can, for example, happen if the state was dependent on another
-// abstract attribute that changed. In every invocation, the update method has
-// to adjust the internal state of an abstract attribute to a point that is
-// justifiable by the underlying IR and the current state of abstract attributes
-// in-flight. Since the IR is given and assumed to be valid, the information
-// derived from it can be assumed to hold. However, information derived from
-// other abstract attributes is conditional on various things. If the justifying
-// state changed, the `updateImpl` has to revisit the situation and potentially
-// find another justification or limit the optimistic assumes made.
-//
-// Change is the key in this framework. Until a state of no-change, thus a
-// fixpoint, is reached, the Attributor will query the abstract attributes
-// in-flight to re-evaluate their state. If the (current) state is too
-// optimistic, hence it cannot be justified anymore through other abstract
-// attributes or the state of the IR, the state of the abstract attribute will
-// have to change. Generally, we assume abstract attribute state to be a finite
-// height lattice and the update function to be monotone. However, these
-// conditions are not enforced because the iteration limit will guarantee
-// termination. If an optimistic fixpoint is reached, or a pessimistic fix
-// point is enforced after a timeout, the abstract attributes are tasked to
-// manifest their result in the IR for passes to come.
-//
-// Attribute manifestation is not mandatory. If desired, there is support to
-// generate a single or multiple LLVM-IR attributes already in the helper struct
-// IRAttribute. In the simplest case, a subclass inherits from IRAttribute with
-// a proper Attribute::AttrKind as template parameter. The Attributor
-// manifestation framework will then create and place a new attribute if it is
-// allowed to do so (based on the abstract state). Other use cases can be
-// achieved by overloading AbstractAttribute or IRAttribute methods.
-//
-//
-// The "mechanics" of adding a new "abstract attribute":
-// - Define a class (transitively) inheriting from AbstractAttribute and one
-// (which could be the same) that (transitively) inherits from AbstractState.
-// For the latter, consider the already available BooleanState and
-// {Inc,Dec,Bit}IntegerState if they fit your needs, e.g., you require only a
-// number tracking or bit-encoding.
-// - Implement all pure methods. Also use overloading if the attribute is not
-// conforming with the "default" behavior: A (set of) LLVM-IR attribute(s) for
-// an argument, call site argument, function return value, or function. See
-// the class and method descriptions for more information on the two
-// "Abstract" classes and their respective methods.
-// - Register opportunities for the new abstract attribute in the
-// `Attributor::identifyDefaultAbstractAttributes` method if it should be
-// counted as a 'default' attribute.
-// - Add sufficient tests.
-// - Add a Statistics object for bookkeeping. If it is a simple (set of)
-// attribute(s) manifested through the Attributor manifestation framework, see
-// the bookkeeping function in Attributor.cpp.
-// - If instructions with a certain opcode are interesting to the attribute, add
-// that opcode to the switch in `Attributor::identifyAbstractAttributes`. This
-// will make it possible to query all those instructions through the
-// `InformationCache::getOpcodeInstMapForFunction` interface and eliminate the
-// need to traverse the IR repeatedly.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
-#define LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
-
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/GraphTraits.h"
-#include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SetOperations.h"
-#include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallSet.h"
-#include "llvm/ADT/iterator.h"
-#include "llvm/Analysis/AssumeBundleQueries.h"
-#include "llvm/Analysis/CFG.h"
-#include "llvm/Analysis/CGSCCPassManager.h"
-#include "llvm/Analysis/LazyCallGraph.h"
-#include "llvm/Analysis/LoopInfo.h"
-#include "llvm/Analysis/MemoryLocation.h"
-#include "llvm/Analysis/MustExecute.h"
-#include "llvm/Analysis/OptimizationRemarkEmitter.h"
-#include "llvm/Analysis/PostDominators.h"
-#include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/IR/AbstractCallSite.h"
-#include "llvm/IR/Attributes.h"
-#include "llvm/IR/ConstantRange.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/InstIterator.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/PassManager.h"
-#include "llvm/IR/Value.h"
-#include "llvm/Support/Alignment.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/DOTGraphTraits.h"
-#include "llvm/Support/DebugCounter.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/ModRef.h"
-#include "llvm/Support/TimeProfiler.h"
-#include "llvm/Support/TypeSize.h"
-#include "llvm/TargetParser/Triple.h"
-#include "llvm/Transforms/Utils/CallGraphUpdater.h"
-
-#include <limits>
-#include <map>
-#include <optional>
-
-namespace llvm {
-
-class DataLayout;
-class LLVMContext;
-class Pass;
-template <typename Fn> class function_ref;
-struct AADepGraphNode;
-struct AADepGraph;
-struct Attributor;
-struct AbstractAttribute;
-struct InformationCache;
-struct AAIsDead;
-struct AttributorCallGraph;
-struct IRPosition;
-
-class Function;
-
-/// Abstract Attribute helper functions.
-namespace AA {
-using InstExclusionSetTy = SmallPtrSet<Instruction *, 4>;
-
-enum class GPUAddressSpace : unsigned {
- Generic = 0,
- Global = 1,
- Shared = 3,
- Constant = 4,
- Local = 5,
-};
-
-/// Return true iff \p M target a GPU (and we can use GPU AS reasoning).
-bool isGPU(const Module &M);
-
-/// Flags to distinguish intra-procedural queries from *potentially*
-/// inter-procedural queries. Not that information can be valid for both and
-/// therefore both bits might be set.
-enum ValueScope : uint8_t {
- Intraprocedural = 1,
- Interprocedural = 2,
- AnyScope = Intraprocedural | Interprocedural,
-};
-
-struct ValueAndContext : public std::pair<Value *, const Instruction *> {
- using Base = std::pair<Value *, const Instruction *>;
- ValueAndContext(const Base &B) : Base(B) {}
- ValueAndContext(Value &V, const Instruction *CtxI) : Base(&V, CtxI) {}
- ValueAndContext(Value &V, const Instruction &CtxI) : Base(&V, &CtxI) {}
-
- Value *getValue() const { return this->first; }
- const Instruction *getCtxI() const { return this->second; }
-};
-
-/// Return true if \p I is a `nosync` instruction. Use generic reasoning and
-/// potentially the corresponding AANoSync.
-bool isNoSyncInst(Attributor &A, const Instruction &I,
- const AbstractAttribute &QueryingAA);
-
-/// Return true if \p V is dynamically unique, that is, there are no two
-/// "instances" of \p V at runtime with different values.
-/// Note: If \p ForAnalysisOnly is set we only check that the Attributor will
-/// never use \p V to represent two "instances" not that \p V could not
-/// technically represent them.
-bool isDynamicallyUnique(Attributor &A, const AbstractAttribute &QueryingAA,
- const Value &V, bool ForAnalysisOnly = true);
-
-/// Return true if \p V is a valid value in \p Scope, that is a constant or an
-/// instruction/argument of \p Scope.
-bool isValidInScope(const Value &V, const Function *Scope);
-
-/// Return true if the value of \p VAC is a valid at the position of \p VAC,
-/// that is a constant, an argument of the same function, or an instruction in
-/// that function that dominates the position.
-bool isValidAtPosition(const ValueAndContext &VAC, InformationCache &InfoCache);
-
-/// Try to convert \p V to type \p Ty without introducing new instructions. If
-/// this is not possible return `nullptr`. Note: this function basically knows
-/// how to cast various constants.
-Value *getWithType(Value &V, Type &Ty);
-
-/// Return the combination of \p A and \p B such that the result is a possible
-/// value of both. \p B is potentially casted to match the type \p Ty or the
-/// type of \p A if \p Ty is null.
-///
-/// Examples:
-/// X + none => X
-/// not_none + undef => not_none
-/// V1 + V2 => nullptr
-std::optional<Value *>
-combineOptionalValuesInAAValueLatice(const std::optional<Value *> &A,
- const std::optional<Value *> &B, Type *Ty);
-
-/// Helper to represent an access offset and size, with logic to deal with
-/// uncertainty and check for overlapping accesses.
-struct RangeTy {
- int64_t Offset = Unassigned;
- int64_t Size = Unassigned;
-
- RangeTy(int64_t Offset, int64_t Size) : Offset(Offset), Size(Size) {}
- RangeTy() = default;
- static RangeTy getUnknown() { return RangeTy{Unknown, Unknown}; }
-
- /// Return true if offset or size are unknown.
- bool offsetOrSizeAreUnknown() const {
- return Offset == RangeTy::Unknown || Size == RangeTy::Unknown;
- }
-
- /// Return true if offset and size are unknown, thus this is the default
- /// unknown object.
- bool offsetAndSizeAreUnknown() const {
- return Offset == RangeTy::Unknown && Size == RangeTy::Unknown;
- }
-
- /// Return true if the offset and size are unassigned.
- bool isUnassigned() const {
- assert((Offset == RangeTy::Unassigned) == (Size == RangeTy::Unassigned) &&
- "Inconsistent state!");
- return Offset == RangeTy::Unassigned;
- }
-
- /// Return true if this offset and size pair might describe an address that
- /// overlaps with \p Range.
- bool mayOverlap(const RangeTy &Range) const {
- // Any unknown value and we are giving up -> overlap.
- if (offsetOrSizeAreUnknown() || Range.offsetOrSizeAreUnknown())
- return true;
-
- // Check if one offset point is in the other interval [offset,
- // offset+size].
- return Range.Offset + Range.Size > Offset && Range.Offset < Offset + Size;
- }
-
- RangeTy &operator&=(const RangeTy &R) {
- if (R.isUnassigned())
- return *this;
- if (isUnassigned())
- return *this = R;
- if (Offset == Unknown || R.Offset == Unknown)
- Offset = Unknown;
- if (Size == Unknown || R.Size == Unknown)
- Size = Unknown;
- if (offsetAndSizeAreUnknown())
- return *this;
- if (Offset == Unknown) {
- Size = std::max(Size, R.Size);
- } else if (Size == Unknown) {
- Offset = std::min(Offset, R.Offset);
- } else {
- Offset = std::min(Offset, R.Offset);
- Size = std::max(Offset + Size, R.Offset + R.Size) - Offset;
- }
- return *this;
- }
-
- /// Comparison for sorting ranges by offset.
- ///
- /// Returns true if the offset \p L is less than that of \p R.
- inline static bool OffsetLessThan(const RangeTy &L, const RangeTy &R) {
- return L.Offset < R.Offset;
- }
-
- /// Constants used to represent special offsets or sizes.
- /// - We cannot assume that Offsets and Size are non-negative.
- /// - The constants should not clash with DenseMapInfo, such as EmptyKey
- /// (INT64_MAX) and TombstoneKey (INT64_MIN).
- /// We use values "in the middle" of the 64 bit range to represent these
- /// special cases.
- static constexpr int64_t Unassigned = std::numeric_limits<int32_t>::min();
- static constexpr int64_t Unknown = std::numeric_limits<int32_t>::max();
-};
-
-inline raw_ostream &operator<<(raw_ostream &OS, const RangeTy &R) {
- OS << "[" << R.Offset << ", " << R.Size << "]";
- return OS;
-}
-
-inline bool operator==(const RangeTy &A, const RangeTy &B) {
- return A.Offset == B.Offset && A.Size == B.Size;
-}
-
-inline bool operator!=(const RangeTy &A, const RangeTy &B) { return !(A == B); }
-
-/// Return the initial value of \p Obj with type \p Ty if that is a constant.
-Constant *getInitialValueForObj(Attributor &A,
- const AbstractAttribute &QueryingAA, Value &Obj,
- Type &Ty, const TargetLibraryInfo *TLI,
- const DataLayout &DL,
- RangeTy *RangePtr = nullptr);
-
-/// Collect all potential values \p LI could read into \p PotentialValues. That
-/// is, the only values read by \p LI are assumed to be known and all are in
-/// \p PotentialValues. \p PotentialValueOrigins will contain all the
-/// instructions that might have put a potential value into \p PotentialValues.
-/// Dependences onto \p QueryingAA are properly tracked, \p
-/// UsedAssumedInformation will inform the caller if assumed information was
-/// used.
-///
-/// \returns True if the assumed potential copies are all in \p PotentialValues,
-/// false if something went wrong and the copies could not be
-/// determined.
-bool getPotentiallyLoadedValues(
- Attributor &A, LoadInst &LI, SmallSetVector<Value *, 4> &PotentialValues,
- SmallSetVector<Instruction *, 4> &PotentialValueOrigins,
- const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation,
- bool OnlyExact = false);
-
-/// Collect all potential values of the one stored by \p SI into
-/// \p PotentialCopies. That is, the only copies that were made via the
-/// store are assumed to be known and all are in \p PotentialCopies. Dependences
-/// onto \p QueryingAA are properly tracked, \p UsedAssumedInformation will
-/// inform the caller if assumed information was used.
-///
-/// \returns True if the assumed potential copies are all in \p PotentialCopies,
-/// false if something went wrong and the copies could not be
-/// determined.
-bool getPotentialCopiesOfStoredValue(
- Attributor &A, StoreInst &SI, SmallSetVector<Value *, 4> &PotentialCopies,
- const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation,
- bool OnlyExact = false);
-
-/// Return true if \p IRP is readonly. This will query respective AAs that
-/// deduce the information and introduce dependences for \p QueryingAA.
-bool isAssumedReadOnly(Attributor &A, const IRPosition &IRP,
- const AbstractAttribute &QueryingAA, bool &IsKnown);
-
-/// Return true if \p IRP is readnone. This will query respective AAs that
-/// deduce the information and introduce dependences for \p QueryingAA.
-bool isAssumedReadNone(Attributor &A, const IRPosition &IRP,
- const AbstractAttribute &QueryingAA, bool &IsKnown);
-
-/// Return true if \p ToI is potentially reachable from \p FromI without running
-/// into any instruction in \p ExclusionSet The two instructions do not need to
-/// be in the same function. \p GoBackwardsCB can be provided to convey domain
-/// knowledge about the "lifespan" the user is interested in. By default, the
-/// callers of \p FromI are checked as well to determine if \p ToI can be
-/// reached. If the query is not interested in callers beyond a certain point,
-/// e.g., a GPU kernel entry or the function containing an alloca, the
-/// \p GoBackwardsCB should return false.
-bool isPotentiallyReachable(
- Attributor &A, const Instruction &FromI, const Instruction &ToI,
- const AbstractAttribute &QueryingAA,
- const AA::InstExclusionSetTy *ExclusionSet = nullptr,
- std::function<bool(const Function &F)> GoBackwardsCB = nullptr);
-
-/// Same as above but it is sufficient to reach any instruction in \p ToFn.
-bool isPotentiallyReachable(
- Attributor &A, const Instruction &FromI, const Function &ToFn,
- const AbstractAttribute &QueryingAA,
- const AA::InstExclusionSetTy *ExclusionSet = nullptr,
- std::function<bool(const Function &F)> GoBackwardsCB = nullptr);
-
-/// Return true if \p Obj is assumed to be a thread local object.
-bool isAssumedThreadLocalObject(Attributor &A, Value &Obj,
- const AbstractAttribute &QueryingAA);
-
-/// Return true if \p I is potentially affected by a barrier.
-bool isPotentiallyAffectedByBarrier(Attributor &A, const Instruction &I,
- const AbstractAttribute &QueryingAA);
-bool isPotentiallyAffectedByBarrier(Attributor &A, ArrayRef<const Value *> Ptrs,
- const AbstractAttribute &QueryingAA,
- const Instruction *CtxI);
-} // namespace AA
-
-template <>
-struct DenseMapInfo<AA::ValueAndContext>
- : public DenseMapInfo<AA::ValueAndContext::Base> {
- using Base = DenseMapInfo<AA::ValueAndContext::Base>;
- static inline AA::ValueAndContext getEmptyKey() {
- return Base::getEmptyKey();
- }
- static inline AA::ValueAndContext getTombstoneKey() {
- return Base::getTombstoneKey();
- }
- static unsigned getHashValue(const AA::ValueAndContext &VAC) {
- return Base::getHashValue(VAC);
- }
-
- static bool isEqual(const AA::ValueAndContext &LHS,
- const AA::ValueAndContext &RHS) {
- return Base::isEqual(LHS, RHS);
- }
-};
-
-template <>
-struct DenseMapInfo<AA::ValueScope> : public DenseMapInfo<unsigned char> {
- using Base = DenseMapInfo<unsigned char>;
- static inline AA::ValueScope getEmptyKey() {
- return AA::ValueScope(Base::getEmptyKey());
- }
- static inline AA::ValueScope getTombstoneKey() {
- return AA::ValueScope(Base::getTombstoneKey());
- }
- static unsigned getHashValue(const AA::ValueScope &S) {
- return Base::getHashValue(S);
- }
-
- static bool isEqual(const AA::ValueScope &LHS, const AA::ValueScope &RHS) {
- return Base::isEqual(LHS, RHS);
- }
-};
-
-template <>
-struct DenseMapInfo<const AA::InstExclusionSetTy *>
- : public DenseMapInfo<void *> {
- using super = DenseMapInfo<void *>;
- static inline const AA::InstExclusionSetTy *getEmptyKey() {
- return static_cast<const AA::InstExclusionSetTy *>(super::getEmptyKey());
- }
- static inline const AA::InstExclusionSetTy *getTombstoneKey() {
- return static_cast<const AA::InstExclusionSetTy *>(
- super::getTombstoneKey());
- }
- static unsigned getHashValue(const AA::InstExclusionSetTy *BES) {
- unsigned H = 0;
- if (BES)
- for (const auto *II : *BES)
- H += DenseMapInfo<const Instruction *>::getHashValue(II);
- return H;
- }
- static bool isEqual(const AA::InstExclusionSetTy *LHS,
- const AA::InstExclusionSetTy *RHS) {
- if (LHS == RHS)
- return true;
- if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
- LHS == getTombstoneKey() || RHS == getTombstoneKey())
- return false;
- auto SizeLHS = LHS ? LHS->size() : 0;
- auto SizeRHS = RHS ? RHS->size() : 0;
- if (SizeLHS != SizeRHS)
- return false;
- if (SizeRHS == 0)
- return true;
- return llvm::set_is_subset(*LHS, *RHS);
- }
-};
-
-/// The value passed to the line option that defines the maximal initialization
-/// chain length.
-extern unsigned MaxInitializationChainLength;
-
-///{
-enum class ChangeStatus {
- CHANGED,
- UNCHANGED,
-};
-
-ChangeStatus operator|(ChangeStatus l, ChangeStatus r);
-ChangeStatus &operator|=(ChangeStatus &l, ChangeStatus r);
-ChangeStatus operator&(ChangeStatus l, ChangeStatus r);
-ChangeStatus &operator&=(ChangeStatus &l, ChangeStatus r);
-
-enum class DepClassTy {
- REQUIRED, ///< The target cannot be valid if the source is not.
- OPTIONAL, ///< The target may be valid if the source is not.
- NONE, ///< Do not track a dependence between source and target.
-};
-///}
-
-/// The data structure for the nodes of a dependency graph
-struct AADepGraphNode {
-public:
- virtual ~AADepGraphNode() = default;
- using DepTy = PointerIntPair<AADepGraphNode *, 1>;
- using DepSetTy = SmallSetVector<DepTy, 2>;
-
-protected:
- /// Set of dependency graph nodes which should be updated if this one
- /// is updated. The bit encodes if it is optional.
- DepSetTy Deps;
-
- static AADepGraphNode *DepGetVal(const DepTy &DT) { return DT.getPointer(); }
- static AbstractAttribute *DepGetValAA(const DepTy &DT) {
- return cast<AbstractAttribute>(DT.getPointer());
- }
-
- operator AbstractAttribute *() { return cast<AbstractAttribute>(this); }
-
-public:
- using iterator = mapped_iterator<DepSetTy::iterator, decltype(&DepGetVal)>;
- using aaiterator =
- mapped_iterator<DepSetTy::iterator, decltype(&DepGetValAA)>;
-
- aaiterator begin() { return aaiterator(Deps.begin(), &DepGetValAA); }
- aaiterator end() { return aaiterator(Deps.end(), &DepGetValAA); }
- iterator child_begin() { return iterator(Deps.begin(), &DepGetVal); }
- iterator child_end() { return iterator(Deps.end(), &DepGetVal); }
-
- void print(raw_ostream &OS) const { print(nullptr, OS); }
- virtual void print(Attributor *, raw_ostream &OS) const {
- OS << "AADepNode Impl\n";
- }
- DepSetTy &getDeps() { return Deps; }
-
- friend struct Attributor;
- friend struct AADepGraph;
-};
-
-/// The data structure for the dependency graph
-///
-/// Note that in this graph if there is an edge from A to B (A -> B),
-/// then it means that B depends on A, and when the state of A is
-/// updated, node B should also be updated
-struct AADepGraph {
- AADepGraph() = default;
- ~AADepGraph() = default;
-
- using DepTy = AADepGraphNode::DepTy;
- static AADepGraphNode *DepGetVal(const DepTy &DT) { return DT.getPointer(); }
- using iterator =
- mapped_iterator<AADepGraphNode::DepSetTy::iterator, decltype(&DepGetVal)>;
-
- /// There is no root node for the dependency graph. But the SCCIterator
- /// requires a single entry point, so we maintain a fake("synthetic") root
- /// node that depends on every node.
- AADepGraphNode SyntheticRoot;
- AADepGraphNode *GetEntryNode() { return &SyntheticRoot; }
-
- iterator begin() { return SyntheticRoot.child_begin(); }
- iterator end() { return SyntheticRoot.child_end(); }
-
- void viewGraph();
-
- /// Dump graph to file
- void dumpGraph();
-
- /// Print dependency graph
- void print();
-};
-
-/// Helper to describe and deal with positions in the LLVM-IR.
-///
-/// A position in the IR is described by an anchor value and an "offset" that
-/// could be the argument number, for call sites and arguments, or an indicator
-/// of the "position kind". The kinds, specified in the Kind enum below, include
-/// the locations in the attribute list, i.a., function scope and return value,
-/// as well as a distinction between call sites and functions. Finally, there
-/// are floating values that do not have a corresponding attribute list
-/// position.
-struct IRPosition {
- // NOTE: In the future this definition can be changed to support recursive
- // functions.
- using CallBaseContext = CallBase;
-
- /// The positions we distinguish in the IR.
- enum Kind : char {
- IRP_INVALID, ///< An invalid position.
- IRP_FLOAT, ///< A position that is not associated with a spot suitable
- ///< for attributes. This could be any value or instruction.
- IRP_RETURNED, ///< An attribute for the function return value.
- IRP_CALL_SITE_RETURNED, ///< An attribute for a call site return value.
- IRP_FUNCTION, ///< An attribute for a function (scope).
- IRP_CALL_SITE, ///< An attribute for a call site (function scope).
- IRP_ARGUMENT, ///< An attribute for a function argument.
- IRP_CALL_SITE_ARGUMENT, ///< An attribute for a call site argument.
- };
-
- /// Default constructor available to create invalid positions implicitly. All
- /// other positions need to be created explicitly through the appropriate
- /// static member function.
- IRPosition() : Enc(nullptr, ENC_VALUE) { verify(); }
-
- /// Create a position describing the value of \p V.
- static const IRPosition value(const Value &V,
- const CallBaseContext *CBContext = nullptr) {
- if (auto *Arg = dyn_cast<Argument>(&V))
- return IRPosition::argument(*Arg, CBContext);
- if (auto *CB = dyn_cast<CallBase>(&V))
- return IRPosition::callsite_returned(*CB);
- return IRPosition(const_cast<Value &>(V), IRP_FLOAT, CBContext);
- }
-
- /// Create a position describing the instruction \p I. This is different from
- /// the value version because call sites are treated as intrusctions rather
- /// than their return value in this function.
- static const IRPosition inst(const Instruction &I,
- const CallBaseContext *CBContext = nullptr) {
- return IRPosition(const_cast<Instruction &>(I), IRP_FLOAT, CBContext);
- }
-
- /// Create a position describing the function scope of \p F.
- /// \p CBContext is used for call base specific analysis.
- static const IRPosition function(const Function &F,
- const CallBaseContext *CBContext = nullptr) {
- return IRPosition(const_cast<Function &>(F), IRP_FUNCTION, CBContext);
- }
-
- /// Create a position describing the returned value of \p F.
- /// \p CBContext is used for call base specific analysis.
- static const IRPosition returned(const Function &F,
- const CallBaseContext *CBContext = nullptr) {
- return IRPosition(const_cast<Function &>(F), IRP_RETURNED, CBContext);
- }
-
- /// Create a position describing the argument \p Arg.
- /// \p CBContext is used for call base specific analysis.
- static const IRPosition argument(const Argument &Arg,
- const CallBaseContext *CBContext = nullptr) {
- return IRPosition(const_cast<Argument &>(Arg), IRP_ARGUMENT, CBContext);
- }
-
- /// Create a position describing the function scope of \p CB.
- static const IRPosition callsite_function(const CallBase &CB) {
- return IRPosition(const_cast<CallBase &>(CB), IRP_CALL_SITE);
- }
-
- /// Create a position describing the returned value of \p CB.
- static const IRPosition callsite_returned(const CallBase &CB) {
- return IRPosition(const_cast<CallBase &>(CB), IRP_CALL_SITE_RETURNED);
- }
-
- /// Create a position describing the argument of \p CB at position \p ArgNo.
- static const IRPosition callsite_argument(const CallBase &CB,
- unsigned ArgNo) {
- return IRPosition(const_cast<Use &>(CB.getArgOperandUse(ArgNo)),
- IRP_CALL_SITE_ARGUMENT);
- }
-
- /// Create a position describing the argument of \p ACS at position \p ArgNo.
- static const IRPosition callsite_argument(AbstractCallSite ACS,
- unsigned ArgNo) {
- if (ACS.getNumArgOperands() <= ArgNo)
- return IRPosition();
- int CSArgNo = ACS.getCallArgOperandNo(ArgNo);
- if (CSArgNo >= 0)
- return IRPosition::callsite_argument(
- cast<CallBase>(*ACS.getInstruction()), CSArgNo);
- return IRPosition();
- }
-
- /// Create a position with function scope matching the "context" of \p IRP.
- /// If \p IRP is a call site (see isAnyCallSitePosition()) then the result
- /// will be a call site position, otherwise the function position of the
- /// associated function.
- static const IRPosition
- function_scope(const IRPosition &IRP,
- const CallBaseContext *CBContext = nullptr) {
- if (IRP.isAnyCallSitePosition()) {
- return IRPosition::callsite_function(
- cast<CallBase>(IRP.getAnchorValue()));
- }
- assert(IRP.getAssociatedFunction());
- return IRPosition::function(*IRP.getAssociatedFunction(), CBContext);
- }
-
- bool operator==(const IRPosition &RHS) const {
- return Enc == RHS.Enc && RHS.CBContext == CBContext;
- }
- bool operator!=(const IRPosition &RHS) const { return !(*this == RHS); }
-
- /// Return the value this abstract attribute is anchored with.
- ///
- /// The anchor value might not be the associated value if the latter is not
- /// sufficient to determine where arguments will be manifested. This is, so
- /// far, only the case for call site arguments as the value is not sufficient
- /// to pinpoint them. Instead, we can use the call site as an anchor.
- Value &getAnchorValue() const {
- switch (getEncodingBits()) {
- case ENC_VALUE:
- case ENC_RETURNED_VALUE:
- case ENC_FLOATING_FUNCTION:
- return *getAsValuePtr();
- case ENC_CALL_SITE_ARGUMENT_USE:
- return *(getAsUsePtr()->getUser());
- default:
- llvm_unreachable("Unkown encoding!");
- };
- }
-
- /// Return the associated function, if any.
- Function *getAssociatedFunction() const {
- if (auto *CB = dyn_cast<CallBase>(&getAnchorValue())) {
- // We reuse the logic that associates callback calles to arguments of a
- // call site here to identify the callback callee as the associated
- // function.
- if (Argument *Arg = getAssociatedArgument())
- return Arg->getParent();
- return dyn_cast_if_present<Function>(
- CB->getCalledOperand()->stripPointerCasts());
- }
- return getAnchorScope();
- }
-
- /// Return the associated argument, if any.
- Argument *getAssociatedArgument() const;
-
- /// Return true if the position refers to a function interface, that is the
- /// function scope, the function return, or an argument.
- bool isFnInterfaceKind() const {
- switch (getPositionKind()) {
- case IRPosition::IRP_FUNCTION:
- case IRPosition::IRP_RETURNED:
- case IRPosition::IRP_ARGUMENT:
- return true;
- default:
- return false;
- }
- }
-
- /// Return true if this is a function or call site position.
- bool isFunctionScope() const {
- switch (getPositionKind()) {
- case IRPosition::IRP_CALL_SITE:
- case IRPosition::IRP_FUNCTION:
- return true;
- default:
- return false;
- };
- }
-
- /// Return the Function surrounding the anchor value.
- Function *getAnchorScope() const {
- Value &V = getAnchorValue();
- if (isa<Function>(V))
- return &cast<Function>(V);
- if (isa<Argument>(V))
- return cast<Argument>(V).getParent();
- if (isa<Instruction>(V))
- return cast<Instruction>(V).getFunction();
- return nullptr;
- }
-
- /// Return the context instruction, if any.
- Instruction *getCtxI() const {
- Value &V = getAnchorValue();
- if (auto *I = dyn_cast<Instruction>(&V))
- return I;
- if (auto *Arg = dyn_cast<Argument>(&V))
- if (!Arg->getParent()->isDeclaration())
- return &Arg->getParent()->getEntryBlock().front();
- if (auto *F = dyn_cast<Function>(&V))
- if (!F->isDeclaration())
- return &(F->getEntryBlock().front());
- return nullptr;
- }
-
- /// Return the value this abstract attribute is associated with.
- Value &getAssociatedValue() const {
- if (getCallSiteArgNo() < 0 || isa<Argument>(&getAnchorValue()))
- return getAnchorValue();
- assert(isa<CallBase>(&getAnchorValue()) && "Expected a call base!");
- return *cast<CallBase>(&getAnchorValue())
- ->getArgOperand(getCallSiteArgNo());
- }
-
- /// Return the type this abstract attribute is associated with.
- Type *getAssociatedType() const {
- if (getPositionKind() == IRPosition::IRP_RETURNED)
- return getAssociatedFunction()->getReturnType();
- return getAssociatedValue().getType();
- }
-
- /// Return the callee argument number of the associated value if it is an
- /// argument or call site argument, otherwise a negative value. In contrast to
- /// `getCallSiteArgNo` this method will always return the "argument number"
- /// from the perspective of the callee. This may not the same as the call site
- /// if this is a callback call.
- int getCalleeArgNo() const {
- return getArgNo(/* CallbackCalleeArgIfApplicable */ true);
- }
-
- /// Return the call site argument number of the associated value if it is an
- /// argument or call site argument, otherwise a negative value. In contrast to
- /// `getCalleArgNo` this method will always return the "operand number" from
- /// the perspective of the call site. This may not the same as the callee
- /// perspective if this is a callback call.
- int getCallSiteArgNo() const {
- return getArgNo(/* CallbackCalleeArgIfApplicable */ false);
- }
-
- /// Return the index in the attribute list for this position.
- unsigned getAttrIdx() const {
- switch (getPositionKind()) {
- case IRPosition::IRP_INVALID:
- case IRPosition::IRP_FLOAT:
- break;
- case IRPosition::IRP_FUNCTION:
- case IRPosition::IRP_CALL_SITE:
- return AttributeList::FunctionIndex;
- case IRPosition::IRP_RETURNED:
- case IRPosition::IRP_CALL_SITE_RETURNED:
- return AttributeList::ReturnIndex;
- case IRPosition::IRP_ARGUMENT:
- return getCalleeArgNo() + AttributeList::FirstArgIndex;
- case IRPosition::IRP_CALL_SITE_ARGUMENT:
- return getCallSiteArgNo() + AttributeList::FirstArgIndex;
- }
- llvm_unreachable(
- "There is no attribute index for a floating or invalid position!");
- }
-
- /// Return the value attributes are attached to.
- Value *getAttrListAnchor() const {
- if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
- return CB;
- return getAssociatedFunction();
- }
-
- /// Return the attributes associated with this function or call site scope.
- AttributeList getAttrList() const {
- if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
- return CB->getAttributes();
- return getAssociatedFunction()->getAttributes();
- }
-
- /// Update the attributes associated with this function or call site scope.
- void setAttrList(const AttributeList &AttrList) const {
- if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
- return CB->setAttributes(AttrList);
- return getAssociatedFunction()->setAttributes(AttrList);
- }
-
- /// Return the number of arguments associated with this function or call site
- /// scope.
- unsigned getNumArgs() const {
- assert((getPositionKind() == IRP_CALL_SITE ||
- getPositionKind() == IRP_FUNCTION) &&
- "Only valid for function/call site positions!");
- if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
- return CB->arg_size();
- return getAssociatedFunction()->arg_size();
- }
-
- /// Return theargument \p ArgNo associated with this function or call site
- /// scope.
- Value *getArg(unsigned ArgNo) const {
- assert((getPositionKind() == IRP_CALL_SITE ||
- getPositionKind() == IRP_FUNCTION) &&
- "Only valid for function/call site positions!");
- if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
- return CB->getArgOperand(ArgNo);
- return getAssociatedFunction()->getArg(ArgNo);
- }
-
- /// Return the associated position kind.
- Kind getPositionKind() const {
- char EncodingBits = getEncodingBits();
- if (EncodingBits == ENC_CALL_SITE_ARGUMENT_USE)
- return IRP_CALL_SITE_ARGUMENT;
- if (EncodingBits == ENC_FLOATING_FUNCTION)
- return IRP_FLOAT;
-
- Value *V = getAsValuePtr();
- if (!V)
- return IRP_INVALID;
- if (isa<Argument>(V))
- return IRP_ARGUMENT;
- if (isa<Function>(V))
- return isReturnPosition(EncodingBits) ? IRP_RETURNED : IRP_FUNCTION;
- if (isa<CallBase>(V))
- return isReturnPosition(EncodingBits) ? IRP_CALL_SITE_RETURNED
- : IRP_CALL_SITE;
- return IRP_FLOAT;
- }
-
- bool isAnyCallSitePosition() const {
- switch (getPositionKind()) {
- case IRPosition::IRP_CALL_SITE:
- case IRPosition::IRP_CALL_SITE_RETURNED:
- case IRPosition::IRP_CALL_SITE_ARGUMENT:
- return true;
- default:
- return false;
- }
- }
-
- /// Return true if the position is an argument or call site argument.
- bool isArgumentPosition() const {
- switch (getPositionKind()) {
- case IRPosition::IRP_ARGUMENT:
- case IRPosition::IRP_CALL_SITE_ARGUMENT:
- return true;
- default:
- return false;
- }
- }
-
- /// Return the same position without the call base context.
- IRPosition stripCallBaseContext() const {
- IRPosition Result = *this;
- Result.CBContext = nullptr;
- return Result;
- }
-
- /// Get the call base context from the position.
- const CallBaseContext *getCallBaseContext() const { return CBContext; }
-
- /// Check if the position has any call base context.
- bool hasCallBaseContext() const { return CBContext != nullptr; }
-
- /// Special DenseMap key values.
- ///
- ///{
- static const IRPosition EmptyKey;
- static const IRPosition TombstoneKey;
- ///}
-
- /// Conversion into a void * to allow reuse of pointer hashing.
- operator void *() const { return Enc.getOpaqueValue(); }
-
-private:
- /// Private constructor for special values only!
- explicit IRPosition(void *Ptr, const CallBaseContext *CBContext = nullptr)
- : CBContext(CBContext) {
- Enc.setFromOpaqueValue(Ptr);
- }
-
- /// IRPosition anchored at \p AnchorVal with kind/argument numbet \p PK.
- explicit IRPosition(Value &AnchorVal, Kind PK,
- const CallBaseContext *CBContext = nullptr)
- : CBContext(CBContext) {
- switch (PK) {
- case IRPosition::IRP_INVALID:
- llvm_unreachable("Cannot create invalid IRP with an anchor value!");
- break;
- case IRPosition::IRP_FLOAT:
- // Special case for floating functions.
- if (isa<Function>(AnchorVal) || isa<CallBase>(AnchorVal))
- Enc = {&AnchorVal, ENC_FLOATING_FUNCTION};
- else
- Enc = {&AnchorVal, ENC_VALUE};
- break;
- case IRPosition::IRP_FUNCTION:
- case IRPosition::IRP_CALL_SITE:
- Enc = {&AnchorVal, ENC_VALUE};
- break;
- case IRPosition::IRP_RETURNED:
- case IRPosition::IRP_CALL_SITE_RETURNED:
- Enc = {&AnchorVal, ENC_RETURNED_VALUE};
- break;
- case IRPosition::IRP_ARGUMENT:
- Enc = {&AnchorVal, ENC_VALUE};
- break;
- case IRPosition::IRP_CALL_SITE_ARGUMENT:
- llvm_unreachable(
- "Cannot create call site argument IRP with an anchor value!");
- break;
- }
- verify();
- }
-
- /// Return the callee argument number of the associated value if it is an
- /// argument or call site argument. See also `getCalleeArgNo` and
- /// `getCallSiteArgNo`.
- int getArgNo(bool CallbackCalleeArgIfApplicable) const {
- if (CallbackCalleeArgIfApplicable)
- if (Argument *Arg = getAssociatedArgument())
- return Arg->getArgNo();
- switch (getPositionKind()) {
- case IRPosition::IRP_ARGUMENT:
- return cast<Argument>(getAsValuePtr())->getArgNo();
- case IRPosition::IRP_CALL_SITE_ARGUMENT: {
- Use &U = *getAsUsePtr();
- return cast<CallBase>(U.getUser())->getArgOperandNo(&U);
- }
- default:
- return -1;
- }
- }
-
- /// IRPosition for the use \p U. The position kind \p PK needs to be
- /// IRP_CALL_SITE_ARGUMENT, the anchor value is the user, the associated value
- /// the used value.
- explicit IRPosition(Use &U, Kind PK) {
- assert(PK == IRP_CALL_SITE_ARGUMENT &&
- "Use constructor is for call site arguments only!");
- Enc = {&U, ENC_CALL_SITE_ARGUMENT_USE};
- verify();
- }
-
- /// Verify internal invariants.
- void verify();
-
- /// Return the underlying pointer as Value *, valid for all positions but
- /// IRP_CALL_SITE_ARGUMENT.
- Value *getAsValuePtr() const {
- assert(getEncodingBits() != ENC_CALL_SITE_ARGUMENT_USE &&
- "Not a value pointer!");
- return reinterpret_cast<Value *>(Enc.getPointer());
- }
-
- /// Return the underlying pointer as Use *, valid only for
- /// IRP_CALL_SITE_ARGUMENT positions.
- Use *getAsUsePtr() const {
- assert(getEncodingBits() == ENC_CALL_SITE_ARGUMENT_USE &&
- "Not a value pointer!");
- return reinterpret_cast<Use *>(Enc.getPointer());
- }
-
- /// Return true if \p EncodingBits describe a returned or call site returned
- /// position.
- static bool isReturnPosition(char EncodingBits) {
- return EncodingBits == ENC_RETURNED_VALUE;
- }
-
- /// Return true if the encoding bits describe a returned or call site returned
- /// position.
- bool isReturnPosition() const { return isReturnPosition(getEncodingBits()); }
-
- /// The encoding of the IRPosition is a combination of a pointer and two
- /// encoding bits. The values of the encoding bits are defined in the enum
- /// below. The pointer is either a Value* (for the first three encoding bit
- /// combinations) or Use* (for ENC_CALL_SITE_ARGUMENT_USE).
- ///
- ///{
- enum {
- ENC_VALUE = 0b00,
- ENC_RETURNED_VALUE = 0b01,
- ENC_FLOATING_FUNCTION = 0b10,
- ENC_CALL_SITE_ARGUMENT_USE = 0b11,
- };
-
- // Reserve the maximal amount of bits so there is no need to mask out the
- // remaining ones. We will not encode anything else in the pointer anyway.
- static constexpr int NumEncodingBits =
- PointerLikeTypeTraits<void *>::NumLowBitsAvailable;
- static_assert(NumEncodingBits >= 2, "At least two bits are required!");
-
- /// The pointer with the encoding bits.
- PointerIntPair<void *, NumEncodingBits, char> Enc;
- ///}
-
- /// Call base context. Used for callsite specific analysis.
- const CallBaseContext *CBContext = nullptr;
-
- /// Return the encoding bits.
- char getEncodingBits() const { return Enc.getInt(); }
-};
-
-/// Helper that allows IRPosition as a key in a DenseMap.
-template <> struct DenseMapInfo<IRPosition> {
- static inline IRPosition getEmptyKey() { return IRPosition::EmptyKey; }
- static inline IRPosition getTombstoneKey() {
- return IRPosition::TombstoneKey;
- }
- static unsigned getHashValue(const IRPosition &IRP) {
- return (DenseMapInfo<void *>::getHashValue(IRP) << 4) ^
- (DenseMapInfo<Value *>::getHashValue(IRP.getCallBaseContext()));
- }
-
- static bool isEqual(const IRPosition &a, const IRPosition &b) {
- return a == b;
- }
-};
-
-/// A visitor class for IR positions.
-///
-/// Given a position P, the SubsumingPositionIterator allows to visit "subsuming
-/// positions" wrt. attributes/information. Thus, if a piece of information
-/// holds for a subsuming position, it also holds for the position P.
-///
-/// The subsuming positions always include the initial position and then,
-/// depending on the position kind, additionally the following ones:
-/// - for IRP_RETURNED:
-/// - the function (IRP_FUNCTION)
-/// - for IRP_ARGUMENT:
-/// - the function (IRP_FUNCTION)
-/// - for IRP_CALL_SITE:
-/// - the callee (IRP_FUNCTION), if known
-/// - for IRP_CALL_SITE_RETURNED:
-/// - the callee (IRP_RETURNED), if known
-/// - the call site (IRP_FUNCTION)
-/// - the callee (IRP_FUNCTION), if known
-/// - for IRP_CALL_SITE_ARGUMENT:
-/// - the argument of the callee (IRP_ARGUMENT), if known
-/// - the callee (IRP_FUNCTION), if known
-/// - the position the call site argument is associated with if it is not
-/// anchored to the call site, e.g., if it is an argument then the argument
-/// (IRP_ARGUMENT)
-class SubsumingPositionIterator {
- SmallVector<IRPosition, 4> IRPositions;
- using iterator = decltype(IRPositions)::iterator;
-
-public:
- SubsumingPositionIterator(const IRPosition &IRP);
- iterator begin() { return IRPositions.begin(); }
- iterator end() { return IRPositions.end(); }
-};
-
-/// Wrapper for FunctionAnalysisManager.
-struct AnalysisGetter {
- // The client may be running the old pass manager, in which case, we need to
- // map the requested Analysis to its equivalent wrapper in the old pass
- // manager. The scheme implemented here does not require every Analysis to be
- // updated. Only those new analyses that the client cares about in the old
- // pass manager need to expose a LegacyWrapper type, and that wrapper should
- // support a getResult() method that matches the new Analysis.
- //
- // We need SFINAE to check for the LegacyWrapper, but function templates don't
- // allow partial specialization, which is needed in this case. So instead, we
- // use a constexpr bool to perform the SFINAE, and then use this information
- // inside the function template.
- template <typename, typename = void>
- static constexpr bool HasLegacyWrapper = false;
-
- template <typename Analysis>
- typename Analysis::Result *getAnalysis(const Function &F,
- bool RequestCachedOnly = false) {
- if (!LegacyPass && !FAM)
- return nullptr;
- if (FAM) {
- if (CachedOnly || RequestCachedOnly)
- return FAM->getCachedResult<Analysis>(const_cast<Function &>(F));
- return &FAM->getResult<Analysis>(const_cast<Function &>(F));
- }
- if constexpr (HasLegacyWrapper<Analysis>) {
- if (!CachedOnly && !RequestCachedOnly)
- return &LegacyPass
- ->getAnalysis<typename Analysis::LegacyWrapper>(
- const_cast<Function &>(F))
- .getResult();
- if (auto *P =
- LegacyPass
- ->getAnalysisIfAvailable<typename Analysis::LegacyWrapper>())
- return &P->getResult();
- }
- return nullptr;
- }
-
- /// Invalidates the analyses. Valid only when using the new pass manager.
- void invalidateAnalyses() {
- assert(FAM && "Can only be used from the new PM!");
- FAM->clear();
- }
-
- AnalysisGetter(FunctionAnalysisManager &FAM, bool CachedOnly = false)
- : FAM(&FAM), CachedOnly(CachedOnly) {}
- AnalysisGetter(Pass *P, bool CachedOnly = false)
- : LegacyPass(P), CachedOnly(CachedOnly) {}
- AnalysisGetter() = default;
-
-private:
- FunctionAnalysisManager *FAM = nullptr;
- Pass *LegacyPass = nullptr;
-
- /// If \p CachedOnly is true, no pass is created, just existing results are
- /// used. Also available per request.
- bool CachedOnly = false;
-};
-
-template <typename Analysis>
-constexpr bool AnalysisGetter::HasLegacyWrapper<
- Analysis, std::void_t<typename Analysis::LegacyWrapper>> = true;
-
-/// Data structure to hold cached (LLVM-IR) information.
-///
-/// All attributes are given an InformationCache object at creation time to
-/// avoid inspection of the IR by all of them individually. This default
-/// InformationCache will hold information required by 'default' attributes,
-/// thus the ones deduced when Attributor::identifyDefaultAbstractAttributes(..)
-/// is called.
-///
-/// If custom abstract attributes, registered manually through
-/// Attributor::registerAA(...), need more information, especially if it is not
-/// reusable, it is advised to inherit from the InformationCache and cast the
-/// instance down in the abstract attributes.
-struct InformationCache {
- InformationCache(const Module &M, AnalysisGetter &AG,
- BumpPtrAllocator &Allocator, SetVector<Function *> *CGSCC,
- bool UseExplorer = true)
- : CGSCC(CGSCC), DL(M.getDataLayout()), Allocator(Allocator), AG(AG),
- TargetTriple(M.getTargetTriple()) {
- if (UseExplorer)
- Explorer = new (Allocator) MustBeExecutedContextExplorer(
- /* ExploreInterBlock */ true, /* ExploreCFGForward */ true,
- /* ExploreCFGBackward */ true,
- /* LIGetter */
- [&](const Function &F) { return AG.getAnalysis<LoopAnalysis>(F); },
- /* DTGetter */
- [&](const Function &F) {
- return AG.getAnalysis<DominatorTreeAnalysis>(F);
- },
- /* PDTGetter */
- [&](const Function &F) {
- return AG.getAnalysis<PostDominatorTreeAnalysis>(F);
- });
- }
-
- ~InformationCache() {
- // The FunctionInfo objects are allocated via a BumpPtrAllocator, we call
- // the destructor manually.
- for (auto &It : FuncInfoMap)
- It.getSecond()->~FunctionInfo();
- // Same is true for the instruction exclusions sets.
- using AA::InstExclusionSetTy;
- for (auto *BES : BESets)
- BES->~InstExclusionSetTy();
- if (Explorer)
- Explorer->~MustBeExecutedContextExplorer();
- }
-
- /// Apply \p CB to all uses of \p F. If \p LookThroughConstantExprUses is
- /// true, constant expression users are not given to \p CB but their uses are
- /// traversed transitively.
- template <typename CBTy>
- static void foreachUse(Function &F, CBTy CB,
- bool LookThroughConstantExprUses = true) {
- SmallVector<Use *, 8> Worklist(make_pointer_range(F.uses()));
-
- for (unsigned Idx = 0; Idx < Worklist.size(); ++Idx) {
- Use &U = *Worklist[Idx];
-
- // Allow use in constant bitcasts and simply look through them.
- if (LookThroughConstantExprUses && isa<ConstantExpr>(U.getUser())) {
- for (Use &CEU : cast<ConstantExpr>(U.getUser())->uses())
- Worklist.push_back(&CEU);
- continue;
- }
-
- CB(U);
- }
- }
-
- /// The CG-SCC the pass is run on, or nullptr if it is a module pass.
- const SetVector<Function *> *const CGSCC = nullptr;
-
- /// A vector type to hold instructions.
- using InstructionVectorTy = SmallVector<Instruction *, 8>;
-
- /// A map type from opcodes to instructions with this opcode.
- using OpcodeInstMapTy = DenseMap<unsigned, InstructionVectorTy *>;
-
- /// Return the map that relates "interesting" opcodes with all instructions
- /// with that opcode in \p F.
- OpcodeInstMapTy &getOpcodeInstMapForFunction(const Function &F) {
- return getFunctionInfo(F).OpcodeInstMap;
- }
-
- /// Return the instructions in \p F that may read or write memory.
- InstructionVectorTy &getReadOrWriteInstsForFunction(const Function &F) {
- return getFunctionInfo(F).RWInsts;
- }
-
- /// Return MustBeExecutedContextExplorer
- MustBeExecutedContextExplorer *getMustBeExecutedContextExplorer() {
- return Explorer;
- }
-
- /// Return TargetLibraryInfo for function \p F.
- TargetLibraryInfo *getTargetLibraryInfoForFunction(const Function &F) {
- return AG.getAnalysis<TargetLibraryAnalysis>(F);
- }
-
- /// Return true if \p Arg is involved in a must-tail call, thus the argument
- /// of the caller or callee.
- bool isInvolvedInMustTailCall(const Argument &Arg) {
- FunctionInfo &FI = getFunctionInfo(*Arg.getParent());
- return FI.CalledViaMustTail || FI.ContainsMustTailCall;
- }
-
- bool isOnlyUsedByAssume(const Instruction &I) const {
- return AssumeOnlyValues.contains(&I);
- }
-
- /// Invalidates the cached analyses. Valid only when using the new pass
- /// manager.
- void invalidateAnalyses() { AG.invalidateAnalyses(); }
-
- /// Return the analysis result from a pass \p AP for function \p F.
- template <typename AP>
- typename AP::Result *getAnalysisResultForFunction(const Function &F,
- bool CachedOnly = false) {
- return AG.getAnalysis<AP>(F, CachedOnly);
- }
-
- /// Return datalayout used in the module.
- const DataLayout &getDL() { return DL; }
-
- /// Return the map conaining all the knowledge we have from `llvm.assume`s.
- const RetainedKnowledgeMap &getKnowledgeMap() const { return KnowledgeMap; }
-
- /// Given \p BES, return a uniqued version.
- const AA::InstExclusionSetTy *
- getOrCreateUniqueBlockExecutionSet(const AA::InstExclusionSetTy *BES) {
- auto It = BESets.find(BES);
- if (It != BESets.end())
- return *It;
- auto *UniqueBES = new (Allocator) AA::InstExclusionSetTy(*BES);
- bool Success = BESets.insert(UniqueBES).second;
- (void)Success;
- assert(Success && "Expected only new entries to be added");
- return UniqueBES;
- }
-
- /// Return true if the stack (llvm::Alloca) can be accessed by other threads.
- bool stackIsAccessibleByOtherThreads() { return !targetIsGPU(); }
-
- /// Return true if the target is a GPU.
- bool targetIsGPU() {
- return TargetTriple.isAMDGPU() || TargetTriple.isNVPTX();
- }
-
- /// Return all functions that might be called indirectly, only valid for
- /// closed world modules (see isClosedWorldModule).
- const ArrayRef<Function *>
- getIndirectlyCallableFunctions(Attributor &A) const;
-
-private:
- struct FunctionInfo {
- ~FunctionInfo();
-
- /// A nested map that remembers all instructions in a function with a
- /// certain instruction opcode (Instruction::getOpcode()).
- OpcodeInstMapTy OpcodeInstMap;
-
- /// A map from functions to their instructions that may read or write
- /// memory.
- InstructionVectorTy RWInsts;
-
- /// Function is called by a `musttail` call.
- bool CalledViaMustTail;
-
- /// Function contains a `musttail` call.
- bool ContainsMustTailCall;
- };
-
- /// A map type from functions to informatio about it.
- DenseMap<const Function *, FunctionInfo *> FuncInfoMap;
-
- /// Return information about the function \p F, potentially by creating it.
- FunctionInfo &getFunctionInfo(const Function &F) {
- FunctionInfo *&FI = FuncInfoMap[&F];
- if (!FI) {
- FI = new (Allocator) FunctionInfo();
- initializeInformationCache(F, *FI);
- }
- return *FI;
- }
-
- /// Vector of functions that might be callable indirectly, i.a., via a
- /// function pointer.
- SmallVector<Function *> IndirectlyCallableFunctions;
-
- /// Initialize the function information cache \p FI for the function \p F.
- ///
- /// This method needs to be called for all function that might be looked at
- /// through the information cache interface *prior* to looking at them.
- void initializeInformationCache(const Function &F, FunctionInfo &FI);
-
- /// The datalayout used in the module.
- const DataLayout &DL;
-
- /// The allocator used to allocate memory, e.g. for `FunctionInfo`s.
- BumpPtrAllocator &Allocator;
-
- /// MustBeExecutedContextExplorer
- MustBeExecutedContextExplorer *Explorer = nullptr;
-
- /// A map with knowledge retained in `llvm.assume` instructions.
- RetainedKnowledgeMap KnowledgeMap;
-
- /// A container for all instructions that are only used by `llvm.assume`.
- SetVector<const Instruction *> AssumeOnlyValues;
-
- /// Cache for block sets to allow reuse.
- DenseSet<const AA::InstExclusionSetTy *> BESets;
-
- /// Getters for analysis.
- AnalysisGetter &AG;
-
- /// Set of inlineable functions
- SmallPtrSet<const Function *, 8> InlineableFunctions;
-
- /// The triple describing the target machine.
- Triple TargetTriple;
-
- /// Give the Attributor access to the members so
- /// Attributor::identifyDefaultAbstractAttributes(...) can initialize them.
- friend struct Attributor;
-};
-
-/// Configuration for the Attributor.
-struct AttributorConfig {
-
- AttributorConfig(CallGraphUpdater &CGUpdater) : CGUpdater(CGUpdater) {}
-
- /// Is the user of the Attributor a module pass or not. This determines what
- /// IR we can look at and modify. If it is a module pass we might deduce facts
- /// outside the initial function set and modify functions outside that set,
- /// but only as part of the optimization of the functions in the initial
- /// function set. For CGSCC passes we can look at the IR of the module slice
- /// but never run any deduction, or perform any modification, outside the
- /// initial function set (which we assume is the SCC).
- bool IsModulePass = true;
-
- /// Flag to determine if we can delete functions or keep dead ones around.
- bool DeleteFns = true;
-
- /// Flag to determine if we rewrite function signatures.
- bool RewriteSignatures = true;
-
- /// Flag to determine if we want to initialize all default AAs for an internal
- /// function marked live. See also: InitializationCallback>
- bool DefaultInitializeLiveInternals = true;
-
- /// Flag to determine if we should skip all liveness checks early on.
- bool UseLiveness = true;
-
- /// Flag to indicate if the entire world is contained in this module, that
- /// is, no outside functions exist.
- bool IsClosedWorldModule = false;
-
- /// Callback function to be invoked on internal functions marked live.
- std::function<void(Attributor &A, const Function &F)> InitializationCallback =
- nullptr;
-
- /// Callback function to determine if an indirect call targets should be made
- /// direct call targets (with an if-cascade).
- std::function<bool(Attributor &A, const AbstractAttribute &AA, CallBase &CB,
- Function &AssummedCallee)>
- IndirectCalleeSpecializationCallback = nullptr;
-
- /// Helper to update an underlying call graph and to delete functions.
- CallGraphUpdater &CGUpdater;
-
- /// If not null, a set limiting the attribute opportunities.
- DenseSet<const char *> *Allowed = nullptr;
-
- /// Maximum number of iterations to run until fixpoint.
- std::optional<unsigned> MaxFixpointIterations;
-
- /// A callback function that returns an ORE object from a Function pointer.
- ///{
- using OptimizationRemarkGetter =
- function_ref<OptimizationRemarkEmitter &(Function *)>;
- OptimizationRemarkGetter OREGetter = nullptr;
- ///}
-
- /// The name of the pass running the attributor, used to emit remarks.
- const char *PassName = nullptr;
-
- using IPOAmendableCBTy = function_ref<bool(const Function &F)>;
- IPOAmendableCBTy IPOAmendableCB;
-};
-
-/// A debug counter to limit the number of AAs created.
-DEBUG_COUNTER(NumAbstractAttributes, "num-abstract-attributes",
- "How many AAs should be initialized");
-
-/// The fixpoint analysis framework that orchestrates the attribute deduction.
-///
-/// The Attributor provides a general abstract analysis framework (guided
-/// fixpoint iteration) as well as helper functions for the deduction of
-/// (LLVM-IR) attributes. However, also other code properties can be deduced,
-/// propagated, and ultimately manifested through the Attributor framework. This
-/// is particularly useful if these properties interact with attributes and a
-/// co-scheduled deduction allows to improve the solution. Even if not, thus if
-/// attributes/properties are completely isolated, they should use the
-/// Attributor framework to reduce the number of fixpoint iteration frameworks
-/// in the code base. Note that the Attributor design makes sure that isolated
-/// attributes are not impacted, in any way, by others derived at the same time
-/// if there is no cross-reasoning performed.
-///
-/// The public facing interface of the Attributor is kept simple and basically
-/// allows abstract attributes to one thing, query abstract attributes
-/// in-flight. There are two reasons to do this:
-/// a) The optimistic state of one abstract attribute can justify an
-/// optimistic state of another, allowing to framework to end up with an
-/// optimistic (=best possible) fixpoint instead of one based solely on
-/// information in the IR.
-/// b) This avoids reimplementing various kinds of lookups, e.g., to check
-/// for existing IR attributes, in favor of a single lookups interface
-/// provided by an abstract attribute subclass.
-///
-/// NOTE: The mechanics of adding a new "concrete" abstract attribute are
-/// described in the file comment.
-struct Attributor {
-
- /// Constructor
- ///
- /// \param Functions The set of functions we are deriving attributes for.
- /// \param InfoCache Cache to hold various information accessible for
- /// the abstract attributes.
- /// \param Configuration The Attributor configuration which determines what
- /// generic features to use.
- Attributor(SetVector<Function *> &Functions, InformationCache &InfoCache,
- AttributorConfig Configuration);
-
- ~Attributor();
-
- /// Run the analyses until a fixpoint is reached or enforced (timeout).
- ///
- /// The attributes registered with this Attributor can be used after as long
- /// as the Attributor is not destroyed (it owns the attributes now).
- ///
- /// \Returns CHANGED if the IR was changed, otherwise UNCHANGED.
- ChangeStatus run();
-
- /// Lookup an abstract attribute of type \p AAType at position \p IRP. While
- /// no abstract attribute is found equivalent positions are checked, see
- /// SubsumingPositionIterator. Thus, the returned abstract attribute
- /// might be anchored at a different position, e.g., the callee if \p IRP is a
- /// call base.
- ///
- /// This method is the only (supported) way an abstract attribute can retrieve
- /// information from another abstract attribute. As an example, take an
- /// abstract attribute that determines the memory access behavior for a
- /// argument (readnone, readonly, ...). It should use `getAAFor` to get the
- /// most optimistic information for other abstract attributes in-flight, e.g.
- /// the one reasoning about the "captured" state for the argument or the one
- /// reasoning on the memory access behavior of the function as a whole.
- ///
- /// If the DepClass enum is set to `DepClassTy::None` the dependence from
- /// \p QueryingAA to the return abstract attribute is not automatically
- /// recorded. This should only be used if the caller will record the
- /// dependence explicitly if necessary, thus if it the returned abstract
- /// attribute is used for reasoning. To record the dependences explicitly use
- /// the `Attributor::recordDependence` method.
- template <typename AAType>
- const AAType *getAAFor(const AbstractAttribute &QueryingAA,
- const IRPosition &IRP, DepClassTy DepClass) {
- return getOrCreateAAFor<AAType>(IRP, &QueryingAA, DepClass,
- /* ForceUpdate */ false);
- }
-
- /// The version of getAAFor that allows to omit a querying abstract
- /// attribute. Using this after Attributor started running is restricted to
- /// only the Attributor itself. Initial seeding of AAs can be done via this
- /// function.
- /// NOTE: ForceUpdate is ignored in any stage other than the update stage.
- template <typename AAType>
- const AAType *getOrCreateAAFor(IRPosition IRP,
- const AbstractAttribute *QueryingAA,
- DepClassTy DepClass, bool ForceUpdate = false,
- bool UpdateAfterInit = true) {
- if (!shouldPropagateCallBaseContext(IRP))
- IRP = IRP.stripCallBaseContext();
-
- if (AAType *AAPtr = lookupAAFor<AAType>(IRP, QueryingAA, DepClass,
- /* AllowInvalidState */ true)) {
- if (ForceUpdate && Phase == AttributorPhase::UPDATE)
- updateAA(*AAPtr);
- return AAPtr;
- }
-
- bool ShouldUpdateAA;
- if (!shouldInitialize<AAType>(IRP, ShouldUpdateAA))
- return nullptr;
-
- if (!DebugCounter::shouldExecute(NumAbstractAttributes))
- return nullptr;
-
- // No matching attribute found, create one.
- // Use the static create method.
- auto &AA = AAType::createForPosition(IRP, *this);
-
- // Always register a new attribute to make sure we clean up the allocated
- // memory properly.
- registerAA(AA);
-
- // If we are currenty seeding attributes, enforce seeding rules.
- if (Phase == AttributorPhase::SEEDING && !shouldSeedAttribute(AA)) {
- AA.getState().indicatePessimisticFixpoint();
- return &AA;
- }
-
- // Bootstrap the new attribute with an initial update to propagate
- // information, e.g., function -> call site.
- {
- TimeTraceScope TimeScope("initialize", [&]() {
- return AA.getName() +
- std::to_string(AA.getIRPosition().getPositionKind());
- });
- ++InitializationChainLength;
- AA.initialize(*this);
- --InitializationChainLength;
- }
-
- if (!ShouldUpdateAA) {
- AA.getState().indicatePessimisticFixpoint();
- return &AA;
- }
-
- // Allow seeded attributes to declare dependencies.
- // Remember the seeding state.
- if (UpdateAfterInit) {
- AttributorPhase OldPhase = Phase;
- Phase = AttributorPhase::UPDATE;
-
- updateAA(AA);
-
- Phase = OldPhase;
- }
-
- if (QueryingAA && AA.getState().isValidState())
- recordDependence(AA, const_cast<AbstractAttribute &>(*QueryingAA),
- DepClass);
- return &AA;
- }
-
- template <typename AAType>
- const AAType *getOrCreateAAFor(const IRPosition &IRP) {
- return getOrCreateAAFor<AAType>(IRP, /* QueryingAA */ nullptr,
- DepClassTy::NONE);
- }
-
- /// Return the attribute of \p AAType for \p IRP if existing and valid. This
- /// also allows non-AA users lookup.
- template <typename AAType>
- AAType *lookupAAFor(const IRPosition &IRP,
- const AbstractAttribute *QueryingAA = nullptr,
- DepClassTy DepClass = DepClassTy::OPTIONAL,
- bool AllowInvalidState = false) {
- static_assert(std::is_base_of<AbstractAttribute, AAType>::value,
- "Cannot query an attribute with a type not derived from "
- "'AbstractAttribute'!");
- // Lookup the abstract attribute of type AAType. If found, return it after
- // registering a dependence of QueryingAA on the one returned attribute.
- AbstractAttribute *AAPtr = AAMap.lookup({&AAType::ID, IRP});
- if (!AAPtr)
- return nullptr;
-
- AAType *AA = static_cast<AAType *>(AAPtr);
-
- // Do not register a dependence on an attribute with an invalid state.
- if (DepClass != DepClassTy::NONE && QueryingAA &&
- AA->getState().isValidState())
- recordDependence(*AA, const_cast<AbstractAttribute &>(*QueryingAA),
- DepClass);
-
- // Return nullptr if this attribute has an invalid state.
- if (!AllowInvalidState && !AA->getState().isValidState())
- return nullptr;
- return AA;
- }
-
- /// Allows a query AA to request an update if a new query was received.
- void registerForUpdate(AbstractAttribute &AA);
-
- /// Explicitly record a dependence from \p FromAA to \p ToAA, that is if
- /// \p FromAA changes \p ToAA should be updated as well.
- ///
- /// This method should be used in conjunction with the `getAAFor` method and
- /// with the DepClass enum passed to the method set to None. This can
- /// be beneficial to avoid false dependences but it requires the users of
- /// `getAAFor` to explicitly record true dependences through this method.
- /// The \p DepClass flag indicates if the dependence is striclty necessary.
- /// That means for required dependences, if \p FromAA changes to an invalid
- /// state, \p ToAA can be moved to a pessimistic fixpoint because it required
- /// information from \p FromAA but none are available anymore.
- void recordDependence(const AbstractAttribute &FromAA,
- const AbstractAttribute &ToAA, DepClassTy DepClass);
-
- /// Introduce a new abstract attribute into the fixpoint analysis.
- ///
- /// Note that ownership of the attribute is given to the Attributor. It will
- /// invoke delete for the Attributor on destruction of the Attributor.
- ///
- /// Attributes are identified by their IR position (AAType::getIRPosition())
- /// and the address of their static member (see AAType::ID).
- template <typename AAType> AAType ®isterAA(AAType &AA) {
- static_assert(std::is_base_of<AbstractAttribute, AAType>::value,
- "Cannot register an attribute with a type not derived from "
- "'AbstractAttribute'!");
- // Put the attribute in the lookup map structure and the container we use to
- // keep track of all attributes.
- const IRPosition &IRP = AA.getIRPosition();
- AbstractAttribute *&AAPtr = AAMap[{&AAType::ID, IRP}];
-
- assert(!AAPtr && "Attribute already in map!");
- AAPtr = &AA;
-
- // Register AA with the synthetic root only before the manifest stage.
- if (Phase == AttributorPhase::SEEDING || Phase == AttributorPhase::UPDATE)
- DG.SyntheticRoot.Deps.insert(
- AADepGraphNode::DepTy(&AA, unsigned(DepClassTy::REQUIRED)));
-
- return AA;
- }
-
- /// Return the internal information cache.
- InformationCache &getInfoCache() { return InfoCache; }
-
- /// Return true if this is a module pass, false otherwise.
- bool isModulePass() const { return Configuration.IsModulePass; }
-
- /// Return true if we should specialize the call site \b CB for the potential
- /// callee \p Fn.
- bool shouldSpecializeCallSiteForCallee(const AbstractAttribute &AA,
- CallBase &CB, Function &Callee) {
- return Configuration.IndirectCalleeSpecializationCallback
- ? Configuration.IndirectCalleeSpecializationCallback(*this, AA,
- CB, Callee)
- : true;
- }
-
- /// Return true if the module contains the whole world, thus, no outside
- /// functions exist.
- bool isClosedWorldModule() const;
-
- /// Return true if we derive attributes for \p Fn
- bool isRunOn(Function &Fn) const { return isRunOn(&Fn); }
- bool isRunOn(Function *Fn) const {
- return Functions.empty() || Functions.count(Fn);
- }
-
- template <typename AAType> bool shouldUpdateAA(const IRPosition &IRP) {
- // If this is queried in the manifest stage, we force the AA to indicate
- // pessimistic fixpoint immediately.
- if (Phase == AttributorPhase::MANIFEST || Phase == AttributorPhase::CLEANUP)
- return false;
-
- Function *AssociatedFn = IRP.getAssociatedFunction();
-
- if (IRP.isAnyCallSitePosition()) {
- // Check if we require a callee but there is none.
- if (!AssociatedFn && AAType::requiresCalleeForCallBase())
- return false;
-
- // Check if we require non-asm but it is inline asm.
- if (AAType::requiresNonAsmForCallBase() &&
- cast<CallBase>(IRP.getAnchorValue()).isInlineAsm())
- return false;
- }
-
- // Check if we require a calles but we can't see all.
- if (AAType::requiresCallersForArgOrFunction())
- if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION ||
- IRP.getPositionKind() == IRPosition::IRP_ARGUMENT)
- if (!AssociatedFn->hasLocalLinkage())
- return false;
-
- if (!AAType::isValidIRPositionForUpdate(*this, IRP))
- return false;
-
- // We update only AAs associated with functions in the Functions set or
- // call sites of them.
- return (!AssociatedFn || isModulePass() || isRunOn(AssociatedFn) ||
- isRunOn(IRP.getAnchorScope()));
- }
-
- template <typename AAType>
- bool shouldInitialize(const IRPosition &IRP, bool &ShouldUpdateAA) {
- if (!AAType::isValidIRPositionForInit(*this, IRP))
- return false;
-
- if (Configuration.Allowed && !Configuration.Allowed->count(&AAType::ID))
- return false;
-
- // For now we skip anything in naked and optnone functions.
- const Function *AnchorFn = IRP.getAnchorScope();
- if (AnchorFn && (AnchorFn->hasFnAttribute(Attribute::Naked) ||
- AnchorFn->hasFnAttribute(Attribute::OptimizeNone)))
- return false;
-
- // Avoid too many nested initializations to prevent a stack overflow.
- if (InitializationChainLength > MaxInitializationChainLength)
- return false;
-
- ShouldUpdateAA = shouldUpdateAA<AAType>(IRP);
-
- return !AAType::hasTrivialInitializer() || ShouldUpdateAA;
- }
-
- /// Determine opportunities to derive 'default' attributes in \p F and create
- /// abstract attribute objects for them.
- ///
- /// \param F The function that is checked for attribute opportunities.
- ///
- /// Note that abstract attribute instances are generally created even if the
- /// IR already contains the information they would deduce. The most important
- /// reason for this is the single interface, the one of the abstract attribute
- /// instance, which can be queried without the need to look at the IR in
- /// various places.
- void identifyDefaultAbstractAttributes(Function &F);
-
- /// Determine whether the function \p F is IPO amendable
- ///
- /// If a function is exactly defined or it has alwaysinline attribute
- /// and is viable to be inlined, we say it is IPO amendable
- bool isFunctionIPOAmendable(const Function &F) {
- return F.hasExactDefinition() || InfoCache.InlineableFunctions.count(&F) ||
- (Configuration.IPOAmendableCB && Configuration.IPOAmendableCB(F));
- }
-
- /// Mark the internal function \p F as live.
- ///
- /// This will trigger the identification and initialization of attributes for
- /// \p F.
- void markLiveInternalFunction(const Function &F) {
- assert(F.hasLocalLinkage() &&
- "Only local linkage is assumed dead initially.");
-
- if (Configuration.DefaultInitializeLiveInternals)
- identifyDefaultAbstractAttributes(const_cast<Function &>(F));
- if (Configuration.InitializationCallback)
- Configuration.InitializationCallback(*this, F);
- }
-
- /// Helper function to remove callsite.
- void removeCallSite(CallInst *CI) {
- if (!CI)
- return;
-
- Configuration.CGUpdater.removeCallSite(*CI);
- }
-
- /// Record that \p U is to be replaces with \p NV after information was
- /// manifested. This also triggers deletion of trivially dead istructions.
- bool changeUseAfterManifest(Use &U, Value &NV) {
- Value *&V = ToBeChangedUses[&U];
- if (V && (V->stripPointerCasts() == NV.stripPointerCasts() ||
- isa_and_nonnull<UndefValue>(V)))
- return false;
- assert((!V || V == &NV || isa<UndefValue>(NV)) &&
- "Use was registered twice for replacement with different values!");
- V = &NV;
- return true;
- }
-
- /// Helper function to replace all uses associated with \p IRP with \p NV.
- /// Return true if there is any change. The flag \p ChangeDroppable indicates
- /// if dropppable uses should be changed too.
- bool changeAfterManifest(const IRPosition IRP, Value &NV,
- bool ChangeDroppable = true) {
- if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT) {
- auto *CB = cast<CallBase>(IRP.getCtxI());
- return changeUseAfterManifest(
- CB->getArgOperandUse(IRP.getCallSiteArgNo()), NV);
- }
- Value &V = IRP.getAssociatedValue();
- auto &Entry = ToBeChangedValues[&V];
- Value *CurNV = get<0>(Entry);
- if (CurNV && (CurNV->stripPointerCasts() == NV.stripPointerCasts() ||
- isa<UndefValue>(CurNV)))
- return false;
- assert((!CurNV || CurNV == &NV || isa<UndefValue>(NV)) &&
- "Value replacement was registered twice with different values!");
- Entry = {&NV, ChangeDroppable};
- return true;
- }
-
- /// Record that \p I is to be replaced with `unreachable` after information
- /// was manifested.
- void changeToUnreachableAfterManifest(Instruction *I) {
- ToBeChangedToUnreachableInsts.insert(I);
- }
-
- /// Record that \p II has at least one dead successor block. This information
- /// is used, e.g., to replace \p II with a call, after information was
- /// manifested.
- void registerInvokeWithDeadSuccessor(InvokeInst &II) {
- InvokeWithDeadSuccessor.insert(&II);
- }
-
- /// Record that \p I is deleted after information was manifested. This also
- /// triggers deletion of trivially dead istructions.
- void deleteAfterManifest(Instruction &I) { ToBeDeletedInsts.insert(&I); }
-
- /// Record that \p BB is deleted after information was manifested. This also
- /// triggers deletion of trivially dead istructions.
- void deleteAfterManifest(BasicBlock &BB) { ToBeDeletedBlocks.insert(&BB); }
-
- // Record that \p BB is added during the manifest of an AA. Added basic blocks
- // are preserved in the IR.
- void registerManifestAddedBasicBlock(BasicBlock &BB) {
- ManifestAddedBlocks.insert(&BB);
- }
-
- /// Record that \p F is deleted after information was manifested.
- void deleteAfterManifest(Function &F) {
- if (Configuration.DeleteFns)
- ToBeDeletedFunctions.insert(&F);
- }
-
- /// Return the attributes of kind \p AK existing in the IR as operand bundles
- /// of an llvm.assume.
- bool getAttrsFromAssumes(const IRPosition &IRP, Attribute::AttrKind AK,
- SmallVectorImpl<Attribute> &Attrs);
-
- /// Return true if any kind in \p AKs existing in the IR at a position that
- /// will affect this one. See also getAttrs(...).
- /// \param IgnoreSubsumingPositions Flag to determine if subsuming positions,
- /// e.g., the function position if this is an
- /// argument position, should be ignored.
- bool hasAttr(const IRPosition &IRP, ArrayRef<Attribute::AttrKind> AKs,
- bool IgnoreSubsumingPositions = false,
- Attribute::AttrKind ImpliedAttributeKind = Attribute::None);
-
- /// Return the attributes of any kind in \p AKs existing in the IR at a
- /// position that will affect this one. While each position can only have a
- /// single attribute of any kind in \p AKs, there are "subsuming" positions
- /// that could have an attribute as well. This method returns all attributes
- /// found in \p Attrs.
- /// \param IgnoreSubsumingPositions Flag to determine if subsuming positions,
- /// e.g., the function position if this is an
- /// argument position, should be ignored.
- void getAttrs(const IRPosition &IRP, ArrayRef<Attribute::AttrKind> AKs,
- SmallVectorImpl<Attribute> &Attrs,
- bool IgnoreSubsumingPositions = false);
-
- /// Remove all \p AttrKinds attached to \p IRP.
- ChangeStatus removeAttrs(const IRPosition &IRP,
- ArrayRef<Attribute::AttrKind> AttrKinds);
- ChangeStatus removeAttrs(const IRPosition &IRP, ArrayRef<StringRef> Attrs);
-
- /// Attach \p DeducedAttrs to \p IRP, if \p ForceReplace is set we do this
- /// even if the same attribute kind was already present.
- ChangeStatus manifestAttrs(const IRPosition &IRP,
- ArrayRef<Attribute> DeducedAttrs,
- bool ForceReplace = false);
-
-private:
- /// Helper to check \p Attrs for \p AK, if not found, check if \p
- /// AAType::isImpliedByIR is true, and if not, create AAType for \p IRP.
- template <Attribute::AttrKind AK, typename AAType>
- void checkAndQueryIRAttr(const IRPosition &IRP, AttributeSet Attrs);
-
- /// Helper to apply \p CB on all attributes of type \p AttrDescs of \p IRP.
- template <typename DescTy>
- ChangeStatus updateAttrMap(const IRPosition &IRP, ArrayRef<DescTy> AttrDescs,
- function_ref<bool(const DescTy &, AttributeSet,
- AttributeMask &, AttrBuilder &)>
- CB);
-
- /// Mapping from functions/call sites to their attributes.
- DenseMap<Value *, AttributeList> AttrsMap;
-
-public:
- /// If \p IRP is assumed to be a constant, return it, if it is unclear yet,
- /// return std::nullopt, otherwise return `nullptr`.
- std::optional<Constant *> getAssumedConstant(const IRPosition &IRP,
- const AbstractAttribute &AA,
- bool &UsedAssumedInformation);
- std::optional<Constant *> getAssumedConstant(const Value &V,
- const AbstractAttribute &AA,
- bool &UsedAssumedInformation) {
- return getAssumedConstant(IRPosition::value(V), AA, UsedAssumedInformation);
- }
-
- /// If \p V is assumed simplified, return it, if it is unclear yet,
- /// return std::nullopt, otherwise return `nullptr`.
- std::optional<Value *> getAssumedSimplified(const IRPosition &IRP,
- const AbstractAttribute &AA,
- bool &UsedAssumedInformation,
- AA::ValueScope S) {
- return getAssumedSimplified(IRP, &AA, UsedAssumedInformation, S);
- }
- std::optional<Value *> getAssumedSimplified(const Value &V,
- const AbstractAttribute &AA,
- bool &UsedAssumedInformation,
- AA::ValueScope S) {
- return getAssumedSimplified(IRPosition::value(V), AA,
- UsedAssumedInformation, S);
- }
-
- /// If \p V is assumed simplified, return it, if it is unclear yet,
- /// return std::nullopt, otherwise return `nullptr`. Same as the public
- /// version except that it can be used without recording dependences on any \p
- /// AA.
- std::optional<Value *> getAssumedSimplified(const IRPosition &V,
- const AbstractAttribute *AA,
- bool &UsedAssumedInformation,
- AA::ValueScope S);
-
- /// Try to simplify \p IRP and in the scope \p S. If successful, true is
- /// returned and all potential values \p IRP can take are put into \p Values.
- /// If the result in \p Values contains select or PHI instructions it means
- /// those could not be simplified to a single value. Recursive calls with
- /// these instructions will yield their respective potential values. If false
- /// is returned no other information is valid.
- bool getAssumedSimplifiedValues(const IRPosition &IRP,
- const AbstractAttribute *AA,
- SmallVectorImpl<AA::ValueAndContext> &Values,
- AA::ValueScope S,
- bool &UsedAssumedInformation,
- bool RecurseForSelectAndPHI = true);
-
- /// Register \p CB as a simplification callback.
- /// `Attributor::getAssumedSimplified` will use these callbacks before
- /// we it will ask `AAValueSimplify`. It is important to ensure this
- /// is called before `identifyDefaultAbstractAttributes`, assuming the
- /// latter is called at all.
- using SimplifictionCallbackTy = std::function<std::optional<Value *>(
- const IRPosition &, const AbstractAttribute *, bool &)>;
- void registerSimplificationCallback(const IRPosition &IRP,
- const SimplifictionCallbackTy &CB) {
- SimplificationCallbacks[IRP].emplace_back(CB);
- }
-
- /// Return true if there is a simplification callback for \p IRP.
- bool hasSimplificationCallback(const IRPosition &IRP) {
- return SimplificationCallbacks.count(IRP);
- }
-
- /// Register \p CB as a simplification callback.
- /// Similar to \p registerSimplificationCallback, the call back will be called
- /// first when we simplify a global variable \p GV.
- using GlobalVariableSimplifictionCallbackTy =
- std::function<std::optional<Constant *>(
- const GlobalVariable &, const AbstractAttribute *, bool &)>;
- void registerGlobalVariableSimplificationCallback(
- const GlobalVariable &GV,
- const GlobalVariableSimplifictionCallbackTy &CB) {
- GlobalVariableSimplificationCallbacks[&GV].emplace_back(CB);
- }
-
- /// Return true if there is a simplification callback for \p GV.
- bool hasGlobalVariableSimplificationCallback(const GlobalVariable &GV) {
- return GlobalVariableSimplificationCallbacks.count(&GV);
- }
-
- /// Return \p std::nullopt if there is no call back registered for \p GV or
- /// the call back is still not sure if \p GV can be simplified. Return \p
- /// nullptr if \p GV can't be simplified.
- std::optional<Constant *>
- getAssumedInitializerFromCallBack(const GlobalVariable &GV,
- const AbstractAttribute *AA,
- bool &UsedAssumedInformation) {
- assert(GlobalVariableSimplificationCallbacks.contains(&GV));
- for (auto &CB : GlobalVariableSimplificationCallbacks.lookup(&GV)) {
- auto SimplifiedGV = CB(GV, AA, UsedAssumedInformation);
- // For now we assume the call back will not return a std::nullopt.
- assert(SimplifiedGV.has_value() && "SimplifiedGV has not value");
- return *SimplifiedGV;
- }
- llvm_unreachable("there must be a callback registered");
- }
-
- using VirtualUseCallbackTy =
- std::function<bool(Attributor &, const AbstractAttribute *)>;
- void registerVirtualUseCallback(const Value &V,
- const VirtualUseCallbackTy &CB) {
- VirtualUseCallbacks[&V].emplace_back(CB);
- }
-
-private:
- /// The vector with all simplification callbacks registered by outside AAs.
- DenseMap<IRPosition, SmallVector<SimplifictionCallbackTy, 1>>
- SimplificationCallbacks;
-
- /// The vector with all simplification callbacks for global variables
- /// registered by outside AAs.
- DenseMap<const GlobalVariable *,
- SmallVector<GlobalVariableSimplifictionCallbackTy, 1>>
- GlobalVariableSimplificationCallbacks;
-
- DenseMap<const Value *, SmallVector<VirtualUseCallbackTy, 1>>
- VirtualUseCallbacks;
-
-public:
- /// Translate \p V from the callee context into the call site context.
- std::optional<Value *>
- translateArgumentToCallSiteContent(std::optional<Value *> V, CallBase &CB,
- const AbstractAttribute &AA,
- bool &UsedAssumedInformation);
-
- /// Return true if \p AA (or its context instruction) is assumed dead.
- ///
- /// If \p LivenessAA is not provided it is queried.
- bool isAssumedDead(const AbstractAttribute &AA, const AAIsDead *LivenessAA,
- bool &UsedAssumedInformation,
- bool CheckBBLivenessOnly = false,
- DepClassTy DepClass = DepClassTy::OPTIONAL);
-
- /// Return true if \p I is assumed dead.
- ///
- /// If \p LivenessAA is not provided it is queried.
- bool isAssumedDead(const Instruction &I, const AbstractAttribute *QueryingAA,
- const AAIsDead *LivenessAA, bool &UsedAssumedInformation,
- bool CheckBBLivenessOnly = false,
- DepClassTy DepClass = DepClassTy::OPTIONAL,
- bool CheckForDeadStore = false);
-
- /// Return true if \p U is assumed dead.
- ///
- /// If \p FnLivenessAA is not provided it is queried.
- bool isAssumedDead(const Use &U, const AbstractAttribute *QueryingAA,
- const AAIsDead *FnLivenessAA, bool &UsedAssumedInformation,
- bool CheckBBLivenessOnly = false,
- DepClassTy DepClass = DepClassTy::OPTIONAL);
-
- /// Return true if \p IRP is assumed dead.
- ///
- /// If \p FnLivenessAA is not provided it is queried.
- bool isAssumedDead(const IRPosition &IRP, const AbstractAttribute *QueryingAA,
- const AAIsDead *FnLivenessAA, bool &UsedAssumedInformation,
- bool CheckBBLivenessOnly = false,
- DepClassTy DepClass = DepClassTy::OPTIONAL);
-
- /// Return true if \p BB is assumed dead.
- ///
- /// If \p LivenessAA is not provided it is queried.
- bool isAssumedDead(const BasicBlock &BB, const AbstractAttribute *QueryingAA,
- const AAIsDead *FnLivenessAA,
- DepClassTy DepClass = DepClassTy::OPTIONAL);
-
- /// Check \p Pred on all potential Callees of \p CB.
- ///
- /// This method will evaluate \p Pred with all potential callees of \p CB as
- /// input and return true if \p Pred does. If some callees might be unknown
- /// this function will return false.
- bool checkForAllCallees(
- function_ref<bool(ArrayRef<const Function *> Callees)> Pred,
- const AbstractAttribute &QueryingAA, const CallBase &CB);
-
- /// Check \p Pred on all (transitive) uses of \p V.
- ///
- /// This method will evaluate \p Pred on all (transitive) uses of the
- /// associated value and return true if \p Pred holds every time.
- /// If uses are skipped in favor of equivalent ones, e.g., if we look through
- /// memory, the \p EquivalentUseCB will be used to give the caller an idea
- /// what original used was replaced by a new one (or new ones). The visit is
- /// cut short if \p EquivalentUseCB returns false and the function will return
- /// false as well.
- bool checkForAllUses(function_ref<bool(const Use &, bool &)> Pred,
- const AbstractAttribute &QueryingAA, const Value &V,
- bool CheckBBLivenessOnly = false,
- DepClassTy LivenessDepClass = DepClassTy::OPTIONAL,
- bool IgnoreDroppableUses = true,
- function_ref<bool(const Use &OldU, const Use &NewU)>
- EquivalentUseCB = nullptr);
-
- /// Emit a remark generically.
- ///
- /// This template function can be used to generically emit a remark. The
- /// RemarkKind should be one of the following:
- /// - OptimizationRemark to indicate a successful optimization attempt
- /// - OptimizationRemarkMissed to report a failed optimization attempt
- /// - OptimizationRemarkAnalysis to provide additional information about an
- /// optimization attempt
- ///
- /// The remark is built using a callback function \p RemarkCB that takes a
- /// RemarkKind as input and returns a RemarkKind.
- template <typename RemarkKind, typename RemarkCallBack>
- void emitRemark(Instruction *I, StringRef RemarkName,
- RemarkCallBack &&RemarkCB) const {
- if (!Configuration.OREGetter)
- return;
-
- Function *F = I->getFunction();
- auto &ORE = Configuration.OREGetter(F);
-
- if (RemarkName.starts_with("OMP"))
- ORE.emit([&]() {
- return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, I))
- << " [" << RemarkName << "]";
- });
- else
- ORE.emit([&]() {
- return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, I));
- });
- }
-
- /// Emit a remark on a function.
- template <typename RemarkKind, typename RemarkCallBack>
- void emitRemark(Function *F, StringRef RemarkName,
- RemarkCallBack &&RemarkCB) const {
- if (!Configuration.OREGetter)
- return;
-
- auto &ORE = Configuration.OREGetter(F);
-
- if (RemarkName.starts_with("OMP"))
- ORE.emit([&]() {
- return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, F))
- << " [" << RemarkName << "]";
- });
- else
- ORE.emit([&]() {
- return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, F));
- });
- }
-
- /// Helper struct used in the communication between an abstract attribute (AA)
- /// that wants to change the signature of a function and the Attributor which
- /// applies the changes. The struct is partially initialized with the
- /// information from the AA (see the constructor). All other members are
- /// provided by the Attributor prior to invoking any callbacks.
- struct ArgumentReplacementInfo {
- /// Callee repair callback type
- ///
- /// The function repair callback is invoked once to rewire the replacement
- /// arguments in the body of the new function. The argument replacement info
- /// is passed, as build from the registerFunctionSignatureRewrite call, as
- /// well as the replacement function and an iteratore to the first
- /// replacement argument.
- using CalleeRepairCBTy = std::function<void(
- const ArgumentReplacementInfo &, Function &, Function::arg_iterator)>;
-
- /// Abstract call site (ACS) repair callback type
- ///
- /// The abstract call site repair callback is invoked once on every abstract
- /// call site of the replaced function (\see ReplacedFn). The callback needs
- /// to provide the operands for the call to the new replacement function.
- /// The number and type of the operands appended to the provided vector
- /// (second argument) is defined by the number and types determined through
- /// the replacement type vector (\see ReplacementTypes). The first argument
- /// is the ArgumentReplacementInfo object registered with the Attributor
- /// through the registerFunctionSignatureRewrite call.
- using ACSRepairCBTy =
- std::function<void(const ArgumentReplacementInfo &, AbstractCallSite,
- SmallVectorImpl<Value *> &)>;
-
- /// Simple getters, see the corresponding members for details.
- ///{
-
- Attributor &getAttributor() const { return A; }
- const Function &getReplacedFn() const { return ReplacedFn; }
- const Argument &getReplacedArg() const { return ReplacedArg; }
- unsigned getNumReplacementArgs() const { return ReplacementTypes.size(); }
- const SmallVectorImpl<Type *> &getReplacementTypes() const {
- return ReplacementTypes;
- }
-
- ///}
-
- private:
- /// Constructor that takes the argument to be replaced, the types of
- /// the replacement arguments, as well as callbacks to repair the call sites
- /// and new function after the replacement happened.
- ArgumentReplacementInfo(Attributor &A, Argument &Arg,
- ArrayRef<Type *> ReplacementTypes,
- CalleeRepairCBTy &&CalleeRepairCB,
- ACSRepairCBTy &&ACSRepairCB)
- : A(A), ReplacedFn(*Arg.getParent()), ReplacedArg(Arg),
- ReplacementTypes(ReplacementTypes.begin(), ReplacementTypes.end()),
- CalleeRepairCB(std::move(CalleeRepairCB)),
- ACSRepairCB(std::move(ACSRepairCB)) {}
-
- /// Reference to the attributor to allow access from the callbacks.
- Attributor &A;
-
- /// The "old" function replaced by ReplacementFn.
- const Function &ReplacedFn;
-
- /// The "old" argument replaced by new ones defined via ReplacementTypes.
- const Argument &ReplacedArg;
-
- /// The types of the arguments replacing ReplacedArg.
- const SmallVector<Type *, 8> ReplacementTypes;
-
- /// Callee repair callback, see CalleeRepairCBTy.
- const CalleeRepairCBTy CalleeRepairCB;
-
- /// Abstract call site (ACS) repair callback, see ACSRepairCBTy.
- const ACSRepairCBTy ACSRepairCB;
-
- /// Allow access to the private members from the Attributor.
- friend struct Attributor;
- };
-
- /// Check if we can rewrite a function signature.
- ///
- /// The argument \p Arg is replaced with new ones defined by the number,
- /// order, and types in \p ReplacementTypes.
- ///
- /// \returns True, if the replacement can be registered, via
- /// registerFunctionSignatureRewrite, false otherwise.
- bool isValidFunctionSignatureRewrite(Argument &Arg,
- ArrayRef<Type *> ReplacementTypes);
-
- /// Register a rewrite for a function signature.
- ///
- /// The argument \p Arg is replaced with new ones defined by the number,
- /// order, and types in \p ReplacementTypes. The rewiring at the call sites is
- /// done through \p ACSRepairCB and at the callee site through
- /// \p CalleeRepairCB.
- ///
- /// \returns True, if the replacement was registered, false otherwise.
- bool registerFunctionSignatureRewrite(
- Argument &Arg, ArrayRef<Type *> ReplacementTypes,
- ArgumentReplacementInfo::CalleeRepairCBTy &&CalleeRepairCB,
- ArgumentReplacementInfo::ACSRepairCBTy &&ACSRepairCB);
-
- /// Check \p Pred on all function call sites.
- ///
- /// This method will evaluate \p Pred on call sites and return
- /// true if \p Pred holds in every call sites. However, this is only possible
- /// all call sites are known, hence the function has internal linkage.
- /// If true is returned, \p UsedAssumedInformation is set if assumed
- /// information was used to skip or simplify potential call sites.
- bool checkForAllCallSites(function_ref<bool(AbstractCallSite)> Pred,
- const AbstractAttribute &QueryingAA,
- bool RequireAllCallSites,
- bool &UsedAssumedInformation);
-
- /// Check \p Pred on all call sites of \p Fn.
- ///
- /// This method will evaluate \p Pred on call sites and return
- /// true if \p Pred holds in every call sites. However, this is only possible
- /// all call sites are known, hence the function has internal linkage.
- /// If true is returned, \p UsedAssumedInformation is set if assumed
- /// information was used to skip or simplify potential call sites.
- bool checkForAllCallSites(function_ref<bool(AbstractCallSite)> Pred,
- const Function &Fn, bool RequireAllCallSites,
- const AbstractAttribute *QueryingAA,
- bool &UsedAssumedInformation,
- bool CheckPotentiallyDead = false);
-
- /// Check \p Pred on all values potentially returned by the function
- /// associated with \p QueryingAA.
- ///
- /// This is the context insensitive version of the method above.
- bool
- checkForAllReturnedValues(function_ref<bool(Value &)> Pred,
- const AbstractAttribute &QueryingAA,
- AA::ValueScope S = AA::ValueScope::Intraprocedural,
- bool RecurseForSelectAndPHI = true);
-
- /// Check \p Pred on all instructions in \p Fn with an opcode present in
- /// \p Opcodes.
- ///
- /// This method will evaluate \p Pred on all instructions with an opcode
- /// present in \p Opcode and return true if \p Pred holds on all of them.
- bool checkForAllInstructions(function_ref<bool(Instruction &)> Pred,
- const Function *Fn,
- const AbstractAttribute *QueryingAA,
- ArrayRef<unsigned> Opcodes,
- bool &UsedAssumedInformation,
- bool CheckBBLivenessOnly = false,
- bool CheckPotentiallyDead = false);
-
- /// Check \p Pred on all instructions with an opcode present in \p Opcodes.
- ///
- /// This method will evaluate \p Pred on all instructions with an opcode
- /// present in \p Opcode and return true if \p Pred holds on all of them.
- bool checkForAllInstructions(function_ref<bool(Instruction &)> Pred,
- const AbstractAttribute &QueryingAA,
- ArrayRef<unsigned> Opcodes,
- bool &UsedAssumedInformation,
- bool CheckBBLivenessOnly = false,
- bool CheckPotentiallyDead = false);
-
- /// Check \p Pred on all call-like instructions (=CallBased derived).
- ///
- /// See checkForAllCallLikeInstructions(...) for more information.
- bool checkForAllCallLikeInstructions(function_ref<bool(Instruction &)> Pred,
- const AbstractAttribute &QueryingAA,
- bool &UsedAssumedInformation,
- bool CheckBBLivenessOnly = false,
- bool CheckPotentiallyDead = false) {
- return checkForAllInstructions(
- Pred, QueryingAA,
- {(unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
- (unsigned)Instruction::Call},
- UsedAssumedInformation, CheckBBLivenessOnly, CheckPotentiallyDead);
- }
-
- /// Check \p Pred on all Read/Write instructions.
- ///
- /// This method will evaluate \p Pred on all instructions that read or write
- /// to memory present in the information cache and return true if \p Pred
- /// holds on all of them.
- bool checkForAllReadWriteInstructions(function_ref<bool(Instruction &)> Pred,
- AbstractAttribute &QueryingAA,
- bool &UsedAssumedInformation);
-
- /// Create a shallow wrapper for \p F such that \p F has internal linkage
- /// afterwards. It also sets the original \p F 's name to anonymous
- ///
- /// A wrapper is a function with the same type (and attributes) as \p F
- /// that will only call \p F and return the result, if any.
- ///
- /// Assuming the declaration of looks like:
- /// rty F(aty0 arg0, ..., atyN argN);
- ///
- /// The wrapper will then look as follows:
- /// rty wrapper(aty0 arg0, ..., atyN argN) {
- /// return F(arg0, ..., argN);
- /// }
- ///
- static void createShallowWrapper(Function &F);
-
- /// Returns true if the function \p F can be internalized. i.e. it has a
- /// compatible linkage.
- static bool isInternalizable(Function &F);
-
- /// Make another copy of the function \p F such that the copied version has
- /// internal linkage afterwards and can be analysed. Then we replace all uses
- /// of the original function to the copied one
- ///
- /// Only non-locally linked functions that have `linkonce_odr` or `weak_odr`
- /// linkage can be internalized because these linkages guarantee that other
- /// definitions with the same name have the same semantics as this one.
- ///
- /// This will only be run if the `attributor-allow-deep-wrappers` option is
- /// set, or if the function is called with \p Force set to true.
- ///
- /// If the function \p F failed to be internalized the return value will be a
- /// null pointer.
- static Function *internalizeFunction(Function &F, bool Force = false);
-
- /// Make copies of each function in the set \p FnSet such that the copied
- /// version has internal linkage afterwards and can be analysed. Then we
- /// replace all uses of the original function to the copied one. The map
- /// \p FnMap contains a mapping of functions to their internalized versions.
- ///
- /// Only non-locally linked functions that have `linkonce_odr` or `weak_odr`
- /// linkage can be internalized because these linkages guarantee that other
- /// definitions with the same name have the same semantics as this one.
- ///
- /// This version will internalize all the functions in the set \p FnSet at
- /// once and then replace the uses. This prevents internalized functions being
- /// called by external functions when there is an internalized version in the
- /// module.
- static bool internalizeFunctions(SmallPtrSetImpl<Function *> &FnSet,
- DenseMap<Function *, Function *> &FnMap);
-
- /// Return the data layout associated with the anchor scope.
- const DataLayout &getDataLayout() const { return InfoCache.DL; }
-
- /// The allocator used to allocate memory, e.g. for `AbstractAttribute`s.
- BumpPtrAllocator &Allocator;
-
- const SmallSetVector<Function *, 8> &getModifiedFunctions() {
- return CGModifiedFunctions;
- }
-
-private:
- /// This method will do fixpoint iteration until fixpoint or the
- /// maximum iteration count is reached.
- ///
- /// If the maximum iteration count is reached, This method will
- /// indicate pessimistic fixpoint on attributes that transitively depend
- /// on attributes that were scheduled for an update.
- void runTillFixpoint();
-
- /// Gets called after scheduling, manifests attributes to the LLVM IR.
- ChangeStatus manifestAttributes();
-
- /// Gets called after attributes have been manifested, cleans up the IR.
- /// Deletes dead functions, blocks and instructions.
- /// Rewrites function signitures and updates the call graph.
- ChangeStatus cleanupIR();
-
- /// Identify internal functions that are effectively dead, thus not reachable
- /// from a live entry point. The functions are added to ToBeDeletedFunctions.
- void identifyDeadInternalFunctions();
-
- /// Run `::update` on \p AA and track the dependences queried while doing so.
- /// Also adjust the state if we know further updates are not necessary.
- ChangeStatus updateAA(AbstractAttribute &AA);
-
- /// Remember the dependences on the top of the dependence stack such that they
- /// may trigger further updates. (\see DependenceStack)
- void rememberDependences();
-
- /// Determine if CallBase context in \p IRP should be propagated.
- bool shouldPropagateCallBaseContext(const IRPosition &IRP);
-
- /// Apply all requested function signature rewrites
- /// (\see registerFunctionSignatureRewrite) and return Changed if the module
- /// was altered.
- ChangeStatus
- rewriteFunctionSignatures(SmallSetVector<Function *, 8> &ModifiedFns);
-
- /// Check if the Attribute \p AA should be seeded.
- /// See getOrCreateAAFor.
- bool shouldSeedAttribute(AbstractAttribute &AA);
-
- /// A nested map to lookup abstract attributes based on the argument position
- /// on the outer level, and the addresses of the static member (AAType::ID) on
- /// the inner level.
- ///{
- using AAMapKeyTy = std::pair<const char *, IRPosition>;
- DenseMap<AAMapKeyTy, AbstractAttribute *> AAMap;
- ///}
-
- /// Map to remember all requested signature changes (= argument replacements).
- DenseMap<Function *, SmallVector<std::unique_ptr<ArgumentReplacementInfo>, 8>>
- ArgumentReplacementMap;
-
- /// The set of functions we are deriving attributes for.
- SetVector<Function *> &Functions;
-
- /// The information cache that holds pre-processed (LLVM-IR) information.
- InformationCache &InfoCache;
-
- /// Abstract Attribute dependency graph
- AADepGraph DG;
-
- /// Set of functions for which we modified the content such that it might
- /// impact the call graph.
- SmallSetVector<Function *, 8> CGModifiedFunctions;
-
- /// Information about a dependence. If FromAA is changed ToAA needs to be
- /// updated as well.
- struct DepInfo {
- const AbstractAttribute *FromAA;
- const AbstractAttribute *ToAA;
- DepClassTy DepClass;
- };
-
- /// The dependence stack is used to track dependences during an
- /// `AbstractAttribute::update` call. As `AbstractAttribute::update` can be
- /// recursive we might have multiple vectors of dependences in here. The stack
- /// size, should be adjusted according to the expected recursion depth and the
- /// inner dependence vector size to the expected number of dependences per
- /// abstract attribute. Since the inner vectors are actually allocated on the
- /// stack we can be generous with their size.
- using DependenceVector = SmallVector<DepInfo, 8>;
- SmallVector<DependenceVector *, 16> DependenceStack;
-
- /// A set to remember the functions we already assume to be live and visited.
- DenseSet<const Function *> VisitedFunctions;
-
- /// Uses we replace with a new value after manifest is done. We will remove
- /// then trivially dead instructions as well.
- SmallMapVector<Use *, Value *, 32> ToBeChangedUses;
-
- /// Values we replace with a new value after manifest is done. We will remove
- /// then trivially dead instructions as well.
- SmallMapVector<Value *, PointerIntPair<Value *, 1, bool>, 32>
- ToBeChangedValues;
-
- /// Instructions we replace with `unreachable` insts after manifest is done.
- SmallSetVector<WeakVH, 16> ToBeChangedToUnreachableInsts;
-
- /// Invoke instructions with at least a single dead successor block.
- SmallSetVector<WeakVH, 16> InvokeWithDeadSuccessor;
-
- /// A flag that indicates which stage of the process we are in. Initially, the
- /// phase is SEEDING. Phase is changed in `Attributor::run()`
- enum class AttributorPhase {
- SEEDING,
- UPDATE,
- MANIFEST,
- CLEANUP,
- } Phase = AttributorPhase::SEEDING;
-
- /// The current initialization chain length. Tracked to avoid stack overflows.
- unsigned InitializationChainLength = 0;
-
- /// Functions, blocks, and instructions we delete after manifest is done.
- ///
- ///{
- SmallPtrSet<BasicBlock *, 8> ManifestAddedBlocks;
- SmallSetVector<Function *, 8> ToBeDeletedFunctions;
- SmallSetVector<BasicBlock *, 8> ToBeDeletedBlocks;
- SmallSetVector<WeakVH, 8> ToBeDeletedInsts;
- ///}
-
- /// Container with all the query AAs that requested an update via
- /// registerForUpdate.
- SmallSetVector<AbstractAttribute *, 16> QueryAAsAwaitingUpdate;
-
- /// User provided configuration for this Attributor instance.
- const AttributorConfig Configuration;
-
- friend AADepGraph;
- friend AttributorCallGraph;
-};
-
-/// An interface to query the internal state of an abstract attribute.
-///
-/// The abstract state is a minimal interface that allows the Attributor to
-/// communicate with the abstract attributes about their internal state without
-/// enforcing or exposing implementation details, e.g., the (existence of an)
-/// underlying lattice.
-///
-/// It is sufficient to be able to query if a state is (1) valid or invalid, (2)
-/// at a fixpoint, and to indicate to the state that (3) an optimistic fixpoint
-/// was reached or (4) a pessimistic fixpoint was enforced.
-///
-/// All methods need to be implemented by the subclass. For the common use case,
-/// a single boolean state or a bit-encoded state, the BooleanState and
-/// {Inc,Dec,Bit}IntegerState classes are already provided. An abstract
-/// attribute can inherit from them to get the abstract state interface and
-/// additional methods to directly modify the state based if needed. See the
-/// class comments for help.
-struct AbstractState {
- virtual ~AbstractState() = default;
-
- /// Return if this abstract state is in a valid state. If false, no
- /// information provided should be used.
- virtual bool isValidState() const = 0;
-
- /// Return if this abstract state is fixed, thus does not need to be updated
- /// if information changes as it cannot change itself.
- virtual bool isAtFixpoint() const = 0;
-
- /// Indicate that the abstract state should converge to the optimistic state.
- ///
- /// This will usually make the optimistically assumed state the known to be
- /// true state.
- ///
- /// \returns ChangeStatus::UNCHANGED as the assumed value should not change.
- virtual ChangeStatus indicateOptimisticFixpoint() = 0;
-
- /// Indicate that the abstract state should converge to the pessimistic state.
- ///
- /// This will usually revert the optimistically assumed state to the known to
- /// be true state.
- ///
- /// \returns ChangeStatus::CHANGED as the assumed value may change.
- virtual ChangeStatus indicatePessimisticFixpoint() = 0;
-};
-
-/// Simple state with integers encoding.
-///
-/// The interface ensures that the assumed bits are always a subset of the known
-/// bits. Users can only add known bits and, except through adding known bits,
-/// they can only remove assumed bits. This should guarantee monotonicity and
-/// thereby the existence of a fixpoint (if used correctly). The fixpoint is
-/// reached when the assumed and known state/bits are equal. Users can
-/// force/inidicate a fixpoint. If an optimistic one is indicated, the known
-/// state will catch up with the assumed one, for a pessimistic fixpoint it is
-/// the other way around.
-template <typename base_ty, base_ty BestState, base_ty WorstState>
-struct IntegerStateBase : public AbstractState {
- using base_t = base_ty;
-
- IntegerStateBase() = default;
- IntegerStateBase(base_t Assumed) : Assumed(Assumed) {}
-
- /// Return the best possible representable state.
- static constexpr base_t getBestState() { return BestState; }
- static constexpr base_t getBestState(const IntegerStateBase &) {
- return getBestState();
- }
-
- /// Return the worst possible representable state.
- static constexpr base_t getWorstState() { return WorstState; }
- static constexpr base_t getWorstState(const IntegerStateBase &) {
- return getWorstState();
- }
-
- /// See AbstractState::isValidState()
- /// NOTE: For now we simply pretend that the worst possible state is invalid.
- bool isValidState() const override { return Assumed != getWorstState(); }
-
- /// See AbstractState::isAtFixpoint()
- bool isAtFixpoint() const override { return Assumed == Known; }
-
- /// See AbstractState::indicateOptimisticFixpoint(...)
- ChangeStatus indicateOptimisticFixpoint() override {
- Known = Assumed;
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractState::indicatePessimisticFixpoint(...)
- ChangeStatus indicatePessimisticFixpoint() override {
- Assumed = Known;
- return ChangeStatus::CHANGED;
- }
-
- /// Return the known state encoding
- base_t getKnown() const { return Known; }
-
- /// Return the assumed state encoding.
- base_t getAssumed() const { return Assumed; }
-
- /// Equality for IntegerStateBase.
- bool
- operator==(const IntegerStateBase<base_t, BestState, WorstState> &R) const {
- return this->getAssumed() == R.getAssumed() &&
- this->getKnown() == R.getKnown();
- }
-
- /// Inequality for IntegerStateBase.
- bool
- operator!=(const IntegerStateBase<base_t, BestState, WorstState> &R) const {
- return !(*this == R);
- }
-
- /// "Clamp" this state with \p R. The result is subtype dependent but it is
- /// intended that only information assumed in both states will be assumed in
- /// this one afterwards.
- void operator^=(const IntegerStateBase<base_t, BestState, WorstState> &R) {
- handleNewAssumedValue(R.getAssumed());
- }
-
- /// "Clamp" this state with \p R. The result is subtype dependent but it is
- /// intended that information known in either state will be known in
- /// this one afterwards.
- void operator+=(const IntegerStateBase<base_t, BestState, WorstState> &R) {
- handleNewKnownValue(R.getKnown());
- }
-
- void operator|=(const IntegerStateBase<base_t, BestState, WorstState> &R) {
- joinOR(R.getAssumed(), R.getKnown());
- }
-
- void operator&=(const IntegerStateBase<base_t, BestState, WorstState> &R) {
- joinAND(R.getAssumed(), R.getKnown());
- }
-
-protected:
- /// Handle a new assumed value \p Value. Subtype dependent.
- virtual void handleNewAssumedValue(base_t Value) = 0;
-
- /// Handle a new known value \p Value. Subtype dependent.
- virtual void handleNewKnownValue(base_t Value) = 0;
-
- /// Handle a value \p Value. Subtype dependent.
- virtual void joinOR(base_t AssumedValue, base_t KnownValue) = 0;
-
- /// Handle a new assumed value \p Value. Subtype dependent.
- virtual void joinAND(base_t AssumedValue, base_t KnownValue) = 0;
-
- /// The known state encoding in an integer of type base_t.
- base_t Known = getWorstState();
-
- /// The assumed state encoding in an integer of type base_t.
- base_t Assumed = getBestState();
-};
-
-/// Specialization of the integer state for a bit-wise encoding.
-template <typename base_ty = uint32_t, base_ty BestState = ~base_ty(0),
- base_ty WorstState = 0>
-struct BitIntegerState
- : public IntegerStateBase<base_ty, BestState, WorstState> {
- using super = IntegerStateBase<base_ty, BestState, WorstState>;
- using base_t = base_ty;
- BitIntegerState() = default;
- BitIntegerState(base_t Assumed) : super(Assumed) {}
-
- /// Return true if the bits set in \p BitsEncoding are "known bits".
- bool isKnown(base_t BitsEncoding = BestState) const {
- return (this->Known & BitsEncoding) == BitsEncoding;
- }
-
- /// Return true if the bits set in \p BitsEncoding are "assumed bits".
- bool isAssumed(base_t BitsEncoding = BestState) const {
- return (this->Assumed & BitsEncoding) == BitsEncoding;
- }
-
- /// Add the bits in \p BitsEncoding to the "known bits".
- BitIntegerState &addKnownBits(base_t Bits) {
- // Make sure we never miss any "known bits".
- this->Assumed |= Bits;
- this->Known |= Bits;
- return *this;
- }
-
- /// Remove the bits in \p BitsEncoding from the "assumed bits" if not known.
- BitIntegerState &removeAssumedBits(base_t BitsEncoding) {
- return intersectAssumedBits(~BitsEncoding);
- }
-
- /// Remove the bits in \p BitsEncoding from the "known bits".
- BitIntegerState &removeKnownBits(base_t BitsEncoding) {
- this->Known = (this->Known & ~BitsEncoding);
- return *this;
- }
-
- /// Keep only "assumed bits" also set in \p BitsEncoding but all known ones.
- BitIntegerState &intersectAssumedBits(base_t BitsEncoding) {
- // Make sure we never lose any "known bits".
- this->Assumed = (this->Assumed & BitsEncoding) | this->Known;
- return *this;
- }
-
-private:
- void handleNewAssumedValue(base_t Value) override {
- intersectAssumedBits(Value);
- }
- void handleNewKnownValue(base_t Value) override { addKnownBits(Value); }
- void joinOR(base_t AssumedValue, base_t KnownValue) override {
- this->Known |= KnownValue;
- this->Assumed |= AssumedValue;
- }
- void joinAND(base_t AssumedValue, base_t KnownValue) override {
- this->Known &= KnownValue;
- this->Assumed &= AssumedValue;
- }
-};
-
-/// Specialization of the integer state for an increasing value, hence ~0u is
-/// the best state and 0 the worst.
-template <typename base_ty = uint32_t, base_ty BestState = ~base_ty(0),
- base_ty WorstState = 0>
-struct IncIntegerState
- : public IntegerStateBase<base_ty, BestState, WorstState> {
- using super = IntegerStateBase<base_ty, BestState, WorstState>;
- using base_t = base_ty;
-
- IncIntegerState() : super() {}
- IncIntegerState(base_t Assumed) : super(Assumed) {}
-
- /// Return the best possible representable state.
- static constexpr base_t getBestState() { return BestState; }
- static constexpr base_t
- getBestState(const IncIntegerState<base_ty, BestState, WorstState> &) {
- return getBestState();
- }
-
- /// Take minimum of assumed and \p Value.
- IncIntegerState &takeAssumedMinimum(base_t Value) {
- // Make sure we never lose "known value".
- this->Assumed = std::max(std::min(this->Assumed, Value), this->Known);
- return *this;
- }
-
- /// Take maximum of known and \p Value.
- IncIntegerState &takeKnownMaximum(base_t Value) {
- // Make sure we never lose "known value".
- this->Assumed = std::max(Value, this->Assumed);
- this->Known = std::max(Value, this->Known);
- return *this;
- }
-
-private:
- void handleNewAssumedValue(base_t Value) override {
- takeAssumedMinimum(Value);
- }
- void handleNewKnownValue(base_t Value) override { takeKnownMaximum(Value); }
- void joinOR(base_t AssumedValue, base_t KnownValue) override {
- this->Known = std::max(this->Known, KnownValue);
- this->Assumed = std::max(this->Assumed, AssumedValue);
- }
- void joinAND(base_t AssumedValue, base_t KnownValue) override {
- this->Known = std::min(this->Known, KnownValue);
- this->Assumed = std::min(this->Assumed, AssumedValue);
- }
-};
-
-/// Specialization of the integer state for a decreasing value, hence 0 is the
-/// best state and ~0u the worst.
-template <typename base_ty = uint32_t>
-struct DecIntegerState : public IntegerStateBase<base_ty, 0, ~base_ty(0)> {
- using base_t = base_ty;
-
- /// Take maximum of assumed and \p Value.
- DecIntegerState &takeAssumedMaximum(base_t Value) {
- // Make sure we never lose "known value".
- this->Assumed = std::min(std::max(this->Assumed, Value), this->Known);
- return *this;
- }
-
- /// Take minimum of known and \p Value.
- DecIntegerState &takeKnownMinimum(base_t Value) {
- // Make sure we never lose "known value".
- this->Assumed = std::min(Value, this->Assumed);
- this->Known = std::min(Value, this->Known);
- return *this;
- }
-
-private:
- void handleNewAssumedValue(base_t Value) override {
- takeAssumedMaximum(Value);
- }
- void handleNewKnownValue(base_t Value) override { takeKnownMinimum(Value); }
- void joinOR(base_t AssumedValue, base_t KnownValue) override {
- this->Assumed = std::min(this->Assumed, KnownValue);
- this->Assumed = std::min(this->Assumed, AssumedValue);
- }
- void joinAND(base_t AssumedValue, base_t KnownValue) override {
- this->Assumed = std::max(this->Assumed, KnownValue);
- this->Assumed = std::max(this->Assumed, AssumedValue);
- }
-};
-
-/// Simple wrapper for a single bit (boolean) state.
-struct BooleanState : public IntegerStateBase<bool, true, false> {
- using super = IntegerStateBase<bool, true, false>;
- using base_t = IntegerStateBase::base_t;
-
- BooleanState() = default;
- BooleanState(base_t Assumed) : super(Assumed) {}
-
- /// Set the assumed value to \p Value but never below the known one.
- void setAssumed(bool Value) { Assumed &= (Known | Value); }
-
- /// Set the known and asssumed value to \p Value.
- void setKnown(bool Value) {
- Known |= Value;
- Assumed |= Value;
- }
-
- /// Return true if the state is assumed to hold.
- bool isAssumed() const { return getAssumed(); }
-
- /// Return true if the state is known to hold.
- bool isKnown() const { return getKnown(); }
-
-private:
- void handleNewAssumedValue(base_t Value) override {
- if (!Value)
- Assumed = Known;
- }
- void handleNewKnownValue(base_t Value) override {
- if (Value)
- Known = (Assumed = Value);
- }
- void joinOR(base_t AssumedValue, base_t KnownValue) override {
- Known |= KnownValue;
- Assumed |= AssumedValue;
- }
- void joinAND(base_t AssumedValue, base_t KnownValue) override {
- Known &= KnownValue;
- Assumed &= AssumedValue;
- }
-};
-
-/// State for an integer range.
-struct IntegerRangeState : public AbstractState {
-
- /// Bitwidth of the associated value.
- uint32_t BitWidth;
-
- /// State representing assumed range, initially set to empty.
- ConstantRange Assumed;
-
- /// State representing known range, initially set to [-inf, inf].
- ConstantRange Known;
-
- IntegerRangeState(uint32_t BitWidth)
- : BitWidth(BitWidth), Assumed(ConstantRange::getEmpty(BitWidth)),
- Known(ConstantRange::getFull(BitWidth)) {}
-
- IntegerRangeState(const ConstantRange &CR)
- : BitWidth(CR.getBitWidth()), Assumed(CR),
- Known(getWorstState(CR.getBitWidth())) {}
-
- /// Return the worst possible representable state.
- static ConstantRange getWorstState(uint32_t BitWidth) {
- return ConstantRange::getFull(BitWidth);
- }
-
- /// Return the best possible representable state.
- static ConstantRange getBestState(uint32_t BitWidth) {
- return ConstantRange::getEmpty(BitWidth);
- }
- static ConstantRange getBestState(const IntegerRangeState &IRS) {
- return getBestState(IRS.getBitWidth());
- }
-
- /// Return associated values' bit width.
- uint32_t getBitWidth() const { return BitWidth; }
-
- /// See AbstractState::isValidState()
- bool isValidState() const override {
- return BitWidth > 0 && !Assumed.isFullSet();
- }
-
- /// See AbstractState::isAtFixpoint()
- bool isAtFixpoint() const override { return Assumed == Known; }
-
- /// See AbstractState::indicateOptimisticFixpoint(...)
- ChangeStatus indicateOptimisticFixpoint() override {
- Known = Assumed;
- return ChangeStatus::CHANGED;
- }
-
- /// See AbstractState::indicatePessimisticFixpoint(...)
- ChangeStatus indicatePessimisticFixpoint() override {
- Assumed = Known;
- return ChangeStatus::CHANGED;
- }
-
- /// Return the known state encoding
- ConstantRange getKnown() const { return Known; }
-
- /// Return the assumed state encoding.
- ConstantRange getAssumed() const { return Assumed; }
-
- /// Unite assumed range with the passed state.
- void unionAssumed(const ConstantRange &R) {
- // Don't lose a known range.
- Assumed = Assumed.unionWith(R).intersectWith(Known);
- }
-
- /// See IntegerRangeState::unionAssumed(..).
- void unionAssumed(const IntegerRangeState &R) {
- unionAssumed(R.getAssumed());
- }
-
- /// Intersect known range with the passed state.
- void intersectKnown(const ConstantRange &R) {
- Assumed = Assumed.intersectWith(R);
- Known = Known.intersectWith(R);
- }
-
- /// See IntegerRangeState::intersectKnown(..).
- void intersectKnown(const IntegerRangeState &R) {
- intersectKnown(R.getKnown());
- }
-
- /// Equality for IntegerRangeState.
- bool operator==(const IntegerRangeState &R) const {
- return getAssumed() == R.getAssumed() && getKnown() == R.getKnown();
- }
-
- /// "Clamp" this state with \p R. The result is subtype dependent but it is
- /// intended that only information assumed in both states will be assumed in
- /// this one afterwards.
- IntegerRangeState operator^=(const IntegerRangeState &R) {
- // NOTE: `^=` operator seems like `intersect` but in this case, we need to
- // take `union`.
- unionAssumed(R);
- return *this;
- }
-
- IntegerRangeState operator&=(const IntegerRangeState &R) {
- // NOTE: `&=` operator seems like `intersect` but in this case, we need to
- // take `union`.
- Known = Known.unionWith(R.getKnown());
- Assumed = Assumed.unionWith(R.getAssumed());
- return *this;
- }
-};
-
-/// Simple state for a set.
-///
-/// This represents a state containing a set of values. The interface supports
-/// modelling sets that contain all possible elements. The state's internal
-/// value is modified using union or intersection operations.
-template <typename BaseTy> struct SetState : public AbstractState {
- /// A wrapper around a set that has semantics for handling unions and
- /// intersections with a "universal" set that contains all elements.
- struct SetContents {
- /// Creates a universal set with no concrete elements or an empty set.
- SetContents(bool Universal) : Universal(Universal) {}
-
- /// Creates a non-universal set with concrete values.
- SetContents(const DenseSet<BaseTy> &Assumptions)
- : Universal(false), Set(Assumptions) {}
-
- SetContents(bool Universal, const DenseSet<BaseTy> &Assumptions)
- : Universal(Universal), Set(Assumptions) {}
-
- const DenseSet<BaseTy> &getSet() const { return Set; }
-
- bool isUniversal() const { return Universal; }
-
- bool empty() const { return Set.empty() && !Universal; }
-
- /// Finds A := A ^ B where A or B could be the "Universal" set which
- /// contains every possible attribute. Returns true if changes were made.
- bool getIntersection(const SetContents &RHS) {
- bool IsUniversal = Universal;
- unsigned Size = Set.size();
-
- // A := A ^ U = A
- if (RHS.isUniversal())
- return false;
-
- // A := U ^ B = B
- if (Universal)
- Set = RHS.getSet();
- else
- set_intersect(Set, RHS.getSet());
-
- Universal &= RHS.isUniversal();
- return IsUniversal != Universal || Size != Set.size();
- }
-
- /// Finds A := A u B where A or B could be the "Universal" set which
- /// contains every possible attribute. returns true if changes were made.
- bool getUnion(const SetContents &RHS) {
- bool IsUniversal = Universal;
- unsigned Size = Set.size();
-
- // A := A u U = U = U u B
- if (!RHS.isUniversal() && !Universal)
- set_union(Set, RHS.getSet());
-
- Universal |= RHS.isUniversal();
- return IsUniversal != Universal || Size != Set.size();
- }
-
- private:
- /// Indicates if this set is "universal", containing every possible element.
- bool Universal;
-
- /// The set of currently active assumptions.
- DenseSet<BaseTy> Set;
- };
-
- SetState() : Known(false), Assumed(true), IsAtFixedpoint(false) {}
-
- /// Initializes the known state with an initial set and initializes the
- /// assumed state as universal.
- SetState(const DenseSet<BaseTy> &Known)
- : Known(Known), Assumed(true), IsAtFixedpoint(false) {}
-
- /// See AbstractState::isValidState()
- bool isValidState() const override { return !Assumed.empty(); }
-
- /// See AbstractState::isAtFixpoint()
- bool isAtFixpoint() const override { return IsAtFixedpoint; }
-
- /// See AbstractState::indicateOptimisticFixpoint(...)
- ChangeStatus indicateOptimisticFixpoint() override {
- IsAtFixedpoint = true;
- Known = Assumed;
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractState::indicatePessimisticFixpoint(...)
- ChangeStatus indicatePessimisticFixpoint() override {
- IsAtFixedpoint = true;
- Assumed = Known;
- return ChangeStatus::CHANGED;
- }
-
- /// Return the known state encoding.
- const SetContents &getKnown() const { return Known; }
-
- /// Return the assumed state encoding.
- const SetContents &getAssumed() const { return Assumed; }
-
- /// Returns if the set state contains the element.
- bool setContains(const BaseTy &Elem) const {
- return Assumed.getSet().contains(Elem) || Known.getSet().contains(Elem);
- }
-
- /// Performs the set intersection between this set and \p RHS. Returns true if
- /// changes were made.
- bool getIntersection(const SetContents &RHS) {
- bool IsUniversal = Assumed.isUniversal();
- unsigned SizeBefore = Assumed.getSet().size();
-
- // Get intersection and make sure that the known set is still a proper
- // subset of the assumed set. A := K u (A ^ R).
- Assumed.getIntersection(RHS);
- Assumed.getUnion(Known);
-
- return SizeBefore != Assumed.getSet().size() ||
- IsUniversal != Assumed.isUniversal();
- }
-
- /// Performs the set union between this set and \p RHS. Returns true if
- /// changes were made.
- bool getUnion(const SetContents &RHS) { return Assumed.getUnion(RHS); }
-
-private:
- /// The set of values known for this state.
- SetContents Known;
-
- /// The set of assumed values for this state.
- SetContents Assumed;
-
- bool IsAtFixedpoint;
-};
-
-/// Helper to tie a abstract state implementation to an abstract attribute.
-template <typename StateTy, typename BaseType, class... Ts>
-struct StateWrapper : public BaseType, public StateTy {
- /// Provide static access to the type of the state.
- using StateType = StateTy;
-
- StateWrapper(const IRPosition &IRP, Ts... Args)
- : BaseType(IRP), StateTy(Args...) {}
-
- /// See AbstractAttribute::getState(...).
- StateType &getState() override { return *this; }
-
- /// See AbstractAttribute::getState(...).
- const StateType &getState() const override { return *this; }
-};
-
-/// Helper class that provides common functionality to manifest IR attributes.
-template <Attribute::AttrKind AK, typename BaseType, typename AAType>
-struct IRAttribute : public BaseType {
- IRAttribute(const IRPosition &IRP) : BaseType(IRP) {}
-
- /// Most boolean IRAttribute AAs don't do anything non-trivial
- /// in their initializers while non-boolean ones often do. Subclasses can
- /// change this.
- static bool hasTrivialInitializer() { return Attribute::isEnumAttrKind(AK); }
-
- /// Compile time access to the IR attribute kind.
- static constexpr Attribute::AttrKind IRAttributeKind = AK;
-
- /// Return true if the IR attribute(s) associated with this AA are implied for
- /// an undef value.
- static bool isImpliedByUndef() { return true; }
-
- /// Return true if the IR attribute(s) associated with this AA are implied for
- /// an poison value.
- static bool isImpliedByPoison() { return true; }
-
- static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind = AK,
- bool IgnoreSubsumingPositions = false) {
- if (AAType::isImpliedByUndef() && isa<UndefValue>(IRP.getAssociatedValue()))
- return true;
- if (AAType::isImpliedByPoison() &&
- isa<PoisonValue>(IRP.getAssociatedValue()))
- return true;
- return A.hasAttr(IRP, {ImpliedAttributeKind}, IgnoreSubsumingPositions,
- ImpliedAttributeKind);
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- if (isa<UndefValue>(this->getIRPosition().getAssociatedValue()))
- return ChangeStatus::UNCHANGED;
- SmallVector<Attribute, 4> DeducedAttrs;
- getDeducedAttributes(A, this->getAnchorValue().getContext(), DeducedAttrs);
- if (DeducedAttrs.empty())
- return ChangeStatus::UNCHANGED;
- return A.manifestAttrs(this->getIRPosition(), DeducedAttrs);
- }
-
- /// Return the kind that identifies the abstract attribute implementation.
- Attribute::AttrKind getAttrKind() const { return AK; }
-
- /// Return the deduced attributes in \p Attrs.
- virtual void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
- SmallVectorImpl<Attribute> &Attrs) const {
- Attrs.emplace_back(Attribute::get(Ctx, getAttrKind()));
- }
-};
-
-/// Base struct for all "concrete attribute" deductions.
-///
-/// The abstract attribute is a minimal interface that allows the Attributor to
-/// orchestrate the abstract/fixpoint analysis. The design allows to hide away
-/// implementation choices made for the subclasses but also to structure their
-/// implementation and simplify the use of other abstract attributes in-flight.
-///
-/// To allow easy creation of new attributes, most methods have default
-/// implementations. The ones that do not are generally straight forward, except
-/// `AbstractAttribute::updateImpl` which is the location of most reasoning
-/// associated with the abstract attribute. The update is invoked by the
-/// Attributor in case the situation used to justify the current optimistic
-/// state might have changed. The Attributor determines this automatically
-/// by monitoring the `Attributor::getAAFor` calls made by abstract attributes.
-///
-/// The `updateImpl` method should inspect the IR and other abstract attributes
-/// in-flight to justify the best possible (=optimistic) state. The actual
-/// implementation is, similar to the underlying abstract state encoding, not
-/// exposed. In the most common case, the `updateImpl` will go through a list of
-/// reasons why its optimistic state is valid given the current information. If
-/// any combination of them holds and is sufficient to justify the current
-/// optimistic state, the method shall return UNCHAGED. If not, the optimistic
-/// state is adjusted to the situation and the method shall return CHANGED.
-///
-/// If the manifestation of the "concrete attribute" deduced by the subclass
-/// differs from the "default" behavior, which is a (set of) LLVM-IR
-/// attribute(s) for an argument, call site argument, function return value, or
-/// function, the `AbstractAttribute::manifest` method should be overloaded.
-///
-/// NOTE: If the state obtained via getState() is INVALID, thus if
-/// AbstractAttribute::getState().isValidState() returns false, no
-/// information provided by the methods of this class should be used.
-/// NOTE: The Attributor currently has certain limitations to what we can do.
-/// As a general rule of thumb, "concrete" abstract attributes should *for
-/// now* only perform "backward" information propagation. That means
-/// optimistic information obtained through abstract attributes should
-/// only be used at positions that precede the origin of the information
-/// with regards to the program flow. More practically, information can
-/// *now* be propagated from instructions to their enclosing function, but
-/// *not* from call sites to the called function. The mechanisms to allow
-/// both directions will be added in the future.
-/// NOTE: The mechanics of adding a new "concrete" abstract attribute are
-/// described in the file comment.
-struct AbstractAttribute : public IRPosition, public AADepGraphNode {
- using StateType = AbstractState;
-
- AbstractAttribute(const IRPosition &IRP) : IRPosition(IRP) {}
-
- /// Virtual destructor.
- virtual ~AbstractAttribute() = default;
-
- /// Compile time access to the IR attribute kind.
- static constexpr Attribute::AttrKind IRAttributeKind = Attribute::None;
-
- /// This function is used to identify if an \p DGN is of type
- /// AbstractAttribute so that the dyn_cast and cast can use such information
- /// to cast an AADepGraphNode to an AbstractAttribute.
- ///
- /// We eagerly return true here because all AADepGraphNodes except for the
- /// Synthethis Node are of type AbstractAttribute
- static bool classof(const AADepGraphNode *DGN) { return true; }
-
- /// Return false if this AA does anything non-trivial (hence not done by
- /// default) in its initializer.
- static bool hasTrivialInitializer() { return false; }
-
- /// Return true if this AA requires a "callee" (or an associted function) for
- /// a call site positon. Default is optimistic to minimize AAs.
- static bool requiresCalleeForCallBase() { return false; }
-
- /// Return true if this AA requires non-asm "callee" for a call site positon.
- static bool requiresNonAsmForCallBase() { return true; }
-
- /// Return true if this AA requires all callees for an argument or function
- /// positon.
- static bool requiresCallersForArgOrFunction() { return false; }
-
- /// Return false if an AA should not be created for \p IRP.
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- return true;
- }
-
- /// Return false if an AA should not be updated for \p IRP.
- static bool isValidIRPositionForUpdate(Attributor &A, const IRPosition &IRP) {
- Function *AssociatedFn = IRP.getAssociatedFunction();
- bool IsFnInterface = IRP.isFnInterfaceKind();
- assert((!IsFnInterface || AssociatedFn) &&
- "Function interface without a function?");
-
- // TODO: Not all attributes require an exact definition. Find a way to
- // enable deduction for some but not all attributes in case the
- // definition might be changed at runtime, see also
- // http://lists.llvm.org/pipermail/llvm-dev/2018-February/121275.html.
- // TODO: We could always determine abstract attributes and if sufficient
- // information was found we could duplicate the functions that do not
- // have an exact definition.
- return !IsFnInterface || A.isFunctionIPOAmendable(*AssociatedFn);
- }
-
- /// Initialize the state with the information in the Attributor \p A.
- ///
- /// This function is called by the Attributor once all abstract attributes
- /// have been identified. It can and shall be used for task like:
- /// - identify existing knowledge in the IR and use it for the "known state"
- /// - perform any work that is not going to change over time, e.g., determine
- /// a subset of the IR, or attributes in-flight, that have to be looked at
- /// in the `updateImpl` method.
- virtual void initialize(Attributor &A) {}
-
- /// A query AA is always scheduled as long as we do updates because it does
- /// lazy computation that cannot be determined to be done from the outside.
- /// However, while query AAs will not be fixed if they do not have outstanding
- /// dependences, we will only schedule them like other AAs. If a query AA that
- /// received a new query it needs to request an update via
- /// `Attributor::requestUpdateForAA`.
- virtual bool isQueryAA() const { return false; }
-
- /// Return the internal abstract state for inspection.
- virtual StateType &getState() = 0;
- virtual const StateType &getState() const = 0;
-
- /// Return an IR position, see struct IRPosition.
- const IRPosition &getIRPosition() const { return *this; };
- IRPosition &getIRPosition() { return *this; };
-
- /// Helper functions, for debug purposes only.
- ///{
- void print(raw_ostream &OS) const { print(nullptr, OS); }
- void print(Attributor *, raw_ostream &OS) const override;
- virtual void printWithDeps(raw_ostream &OS) const;
- void dump() const { this->print(dbgs()); }
-
- /// This function should return the "summarized" assumed state as string.
- virtual const std::string getAsStr(Attributor *A) const = 0;
-
- /// This function should return the name of the AbstractAttribute
- virtual const std::string getName() const = 0;
-
- /// This function should return the address of the ID of the AbstractAttribute
- virtual const char *getIdAddr() const = 0;
- ///}
-
- /// Allow the Attributor access to the protected methods.
- friend struct Attributor;
-
-protected:
- /// Hook for the Attributor to trigger an update of the internal state.
- ///
- /// If this attribute is already fixed, this method will return UNCHANGED,
- /// otherwise it delegates to `AbstractAttribute::updateImpl`.
- ///
- /// \Return CHANGED if the internal state changed, otherwise UNCHANGED.
- ChangeStatus update(Attributor &A);
-
- /// Hook for the Attributor to trigger the manifestation of the information
- /// represented by the abstract attribute in the LLVM-IR.
- ///
- /// \Return CHANGED if the IR was altered, otherwise UNCHANGED.
- virtual ChangeStatus manifest(Attributor &A) {
- return ChangeStatus::UNCHANGED;
- }
-
- /// Hook to enable custom statistic tracking, called after manifest that
- /// resulted in a change if statistics are enabled.
- ///
- /// We require subclasses to provide an implementation so we remember to
- /// add statistics for them.
- virtual void trackStatistics() const = 0;
-
- /// The actual update/transfer function which has to be implemented by the
- /// derived classes.
- ///
- /// If it is called, the environment has changed and we have to determine if
- /// the current information is still valid or adjust it otherwise.
- ///
- /// \Return CHANGED if the internal state changed, otherwise UNCHANGED.
- virtual ChangeStatus updateImpl(Attributor &A) = 0;
-};
-
-/// Forward declarations of output streams for debug purposes.
-///
-///{
-raw_ostream &operator<<(raw_ostream &OS, const AbstractAttribute &AA);
-raw_ostream &operator<<(raw_ostream &OS, ChangeStatus S);
-raw_ostream &operator<<(raw_ostream &OS, IRPosition::Kind);
-raw_ostream &operator<<(raw_ostream &OS, const IRPosition &);
-raw_ostream &operator<<(raw_ostream &OS, const AbstractState &State);
-template <typename base_ty, base_ty BestState, base_ty WorstState>
-raw_ostream &
-operator<<(raw_ostream &OS,
- const IntegerStateBase<base_ty, BestState, WorstState> &S) {
- return OS << "(" << S.getKnown() << "-" << S.getAssumed() << ")"
- << static_cast<const AbstractState &>(S);
-}
-raw_ostream &operator<<(raw_ostream &OS, const IntegerRangeState &State);
-///}
-
-struct AttributorPass : public PassInfoMixin<AttributorPass> {
- PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-};
-struct AttributorCGSCCPass : public PassInfoMixin<AttributorCGSCCPass> {
- PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
- LazyCallGraph &CG, CGSCCUpdateResult &UR);
-};
-
-/// A more lightweight version of the Attributor which only runs attribute
-/// inference but no simplifications.
-struct AttributorLightPass : public PassInfoMixin<AttributorLightPass> {
- PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-};
-
-/// A more lightweight version of the Attributor which only runs attribute
-/// inference but no simplifications.
-struct AttributorLightCGSCCPass
- : public PassInfoMixin<AttributorLightCGSCCPass> {
- PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
- LazyCallGraph &CG, CGSCCUpdateResult &UR);
-};
-
-/// Helper function to clamp a state \p S of type \p StateType with the
-/// information in \p R and indicate/return if \p S did change (as-in update is
-/// required to be run again).
-template <typename StateType>
-ChangeStatus clampStateAndIndicateChange(StateType &S, const StateType &R) {
- auto Assumed = S.getAssumed();
- S ^= R;
- return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
-}
-
-/// ----------------------------------------------------------------------------
-/// Abstract Attribute Classes
-/// ----------------------------------------------------------------------------
-
-struct AANoUnwind
- : public IRAttribute<Attribute::NoUnwind,
- StateWrapper<BooleanState, AbstractAttribute>,
- AANoUnwind> {
- AANoUnwind(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// Returns true if nounwind is assumed.
- bool isAssumedNoUnwind() const { return getAssumed(); }
-
- /// Returns true if nounwind is known.
- bool isKnownNoUnwind() const { return getKnown(); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANoUnwind &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANoUnwind"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AANoUnwind
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-struct AANoSync
- : public IRAttribute<Attribute::NoSync,
- StateWrapper<BooleanState, AbstractAttribute>,
- AANoSync> {
- AANoSync(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions = false) {
- // Note: This is also run for non-IPO amendable functions.
- assert(ImpliedAttributeKind == Attribute::NoSync);
- if (A.hasAttr(IRP, {Attribute::NoSync}, IgnoreSubsumingPositions,
- Attribute::NoSync))
- return true;
-
- // Check for readonly + non-convergent.
- // TODO: We should be able to use hasAttr for Attributes, not only
- // AttrKinds.
- Function *F = IRP.getAssociatedFunction();
- if (!F || F->isConvergent())
- return false;
-
- SmallVector<Attribute, 2> Attrs;
- A.getAttrs(IRP, {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
-
- MemoryEffects ME = MemoryEffects::unknown();
- for (const Attribute &Attr : Attrs)
- ME &= Attr.getMemoryEffects();
-
- if (!ME.onlyReadsMemory())
- return false;
-
- A.manifestAttrs(IRP, Attribute::get(F->getContext(), Attribute::NoSync));
- return true;
- }
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.isFunctionScope() &&
- !IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return IRAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// Returns true if "nosync" is assumed.
- bool isAssumedNoSync() const { return getAssumed(); }
-
- /// Returns true if "nosync" is known.
- bool isKnownNoSync() const { return getKnown(); }
-
- /// Helper function used to determine whether an instruction is non-relaxed
- /// atomic. In other words, if an atomic instruction does not have unordered
- /// or monotonic ordering
- static bool isNonRelaxedAtomic(const Instruction *I);
-
- /// Helper function specific for intrinsics which are potentially volatile.
- static bool isNoSyncIntrinsic(const Instruction *I);
-
- /// Helper function to determine if \p CB is an aligned (GPU) barrier. Aligned
- /// barriers have to be executed by all threads. The flag \p ExecutedAligned
- /// indicates if the call is executed by all threads in a (thread) block in an
- /// aligned way. If that is the case, non-aligned barriers are effectively
- /// aligned barriers.
- static bool isAlignedBarrier(const CallBase &CB, bool ExecutedAligned);
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANoSync &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANoSync"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AANoSync
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for all nonnull attributes.
-struct AAMustProgress
- : public IRAttribute<Attribute::MustProgress,
- StateWrapper<BooleanState, AbstractAttribute>,
- AAMustProgress> {
- AAMustProgress(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions = false) {
- // Note: This is also run for non-IPO amendable functions.
- assert(ImpliedAttributeKind == Attribute::MustProgress);
- return A.hasAttr(IRP, {Attribute::MustProgress, Attribute::WillReturn},
- IgnoreSubsumingPositions, Attribute::MustProgress);
- }
-
- /// Return true if we assume that the underlying value is nonnull.
- bool isAssumedMustProgress() const { return getAssumed(); }
-
- /// Return true if we know that underlying value is nonnull.
- bool isKnownMustProgress() const { return getKnown(); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAMustProgress &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAMustProgress"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAMustProgress
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for all nonnull attributes.
-struct AANonNull
- : public IRAttribute<Attribute::NonNull,
- StateWrapper<BooleanState, AbstractAttribute>,
- AANonNull> {
- AANonNull(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// See AbstractAttribute::hasTrivialInitializer.
- static bool hasTrivialInitializer() { return false; }
-
- /// See IRAttribute::isImpliedByUndef.
- /// Undef is not necessarily nonnull as nonnull + noundef would cause poison.
- /// Poison implies nonnull though.
- static bool isImpliedByUndef() { return false; }
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return IRAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// See AbstractAttribute::isImpliedByIR(...).
- static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions = false);
-
- /// Return true if we assume that the underlying value is nonnull.
- bool isAssumedNonNull() const { return getAssumed(); }
-
- /// Return true if we know that underlying value is nonnull.
- bool isKnownNonNull() const { return getKnown(); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANonNull &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANonNull"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AANonNull
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract attribute for norecurse.
-struct AANoRecurse
- : public IRAttribute<Attribute::NoRecurse,
- StateWrapper<BooleanState, AbstractAttribute>,
- AANoRecurse> {
- AANoRecurse(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// Return true if "norecurse" is assumed.
- bool isAssumedNoRecurse() const { return getAssumed(); }
-
- /// Return true if "norecurse" is known.
- bool isKnownNoRecurse() const { return getKnown(); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANoRecurse &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANoRecurse"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AANoRecurse
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract attribute for willreturn.
-struct AAWillReturn
- : public IRAttribute<Attribute::WillReturn,
- StateWrapper<BooleanState, AbstractAttribute>,
- AAWillReturn> {
- AAWillReturn(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions = false) {
- // Note: This is also run for non-IPO amendable functions.
- assert(ImpliedAttributeKind == Attribute::WillReturn);
- if (IRAttribute::isImpliedByIR(A, IRP, ImpliedAttributeKind,
- IgnoreSubsumingPositions))
- return true;
- if (!isImpliedByMustprogressAndReadonly(A, IRP))
- return false;
- A.manifestAttrs(IRP, Attribute::get(IRP.getAnchorValue().getContext(),
- Attribute::WillReturn));
- return true;
- }
-
- /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
- static bool isImpliedByMustprogressAndReadonly(Attributor &A,
- const IRPosition &IRP) {
- // Check for `mustprogress` in the scope and the associated function which
- // might be different if this is a call site.
- if (!A.hasAttr(IRP, {Attribute::MustProgress}))
- return false;
-
- SmallVector<Attribute, 2> Attrs;
- A.getAttrs(IRP, {Attribute::Memory}, Attrs,
- /* IgnoreSubsumingPositions */ false);
-
- MemoryEffects ME = MemoryEffects::unknown();
- for (const Attribute &Attr : Attrs)
- ME &= Attr.getMemoryEffects();
- return ME.onlyReadsMemory();
- }
-
- /// Return true if "willreturn" is assumed.
- bool isAssumedWillReturn() const { return getAssumed(); }
-
- /// Return true if "willreturn" is known.
- bool isKnownWillReturn() const { return getKnown(); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAWillReturn &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAWillReturn"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AAWillReturn
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract attribute for undefined behavior.
-struct AAUndefinedBehavior
- : public StateWrapper<BooleanState, AbstractAttribute> {
- using Base = StateWrapper<BooleanState, AbstractAttribute>;
- AAUndefinedBehavior(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// Return true if "undefined behavior" is assumed.
- bool isAssumedToCauseUB() const { return getAssumed(); }
-
- /// Return true if "undefined behavior" is assumed for a specific instruction.
- virtual bool isAssumedToCauseUB(Instruction *I) const = 0;
-
- /// Return true if "undefined behavior" is known.
- bool isKnownToCauseUB() const { return getKnown(); }
-
- /// Return true if "undefined behavior" is known for a specific instruction.
- virtual bool isKnownToCauseUB(Instruction *I) const = 0;
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAUndefinedBehavior &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAUndefinedBehavior"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAUndefineBehavior
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface to determine reachability of point A to B.
-struct AAIntraFnReachability
- : public StateWrapper<BooleanState, AbstractAttribute> {
- using Base = StateWrapper<BooleanState, AbstractAttribute>;
- AAIntraFnReachability(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// Returns true if 'From' instruction is assumed to reach, 'To' instruction.
- /// Users should provide two positions they are interested in, and the class
- /// determines (and caches) reachability.
- virtual bool isAssumedReachable(
- Attributor &A, const Instruction &From, const Instruction &To,
- const AA::InstExclusionSetTy *ExclusionSet = nullptr) const = 0;
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAIntraFnReachability &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAIntraFnReachability"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAIntraFnReachability
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for all noalias attributes.
-struct AANoAlias
- : public IRAttribute<Attribute::NoAlias,
- StateWrapper<BooleanState, AbstractAttribute>,
- AANoAlias> {
- AANoAlias(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return IRAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// See IRAttribute::isImpliedByIR
- static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions = false);
-
- /// See AbstractAttribute::requiresCallersForArgOrFunction
- static bool requiresCallersForArgOrFunction() { return true; }
-
- /// Return true if we assume that the underlying value is alias.
- bool isAssumedNoAlias() const { return getAssumed(); }
-
- /// Return true if we know that underlying value is noalias.
- bool isKnownNoAlias() const { return getKnown(); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANoAlias &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANoAlias"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AANoAlias
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An AbstractAttribute for nofree.
-struct AANoFree
- : public IRAttribute<Attribute::NoFree,
- StateWrapper<BooleanState, AbstractAttribute>,
- AANoFree> {
- AANoFree(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// See IRAttribute::isImpliedByIR
- static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions = false) {
- // Note: This is also run for non-IPO amendable functions.
- assert(ImpliedAttributeKind == Attribute::NoFree);
- return A.hasAttr(
- IRP, {Attribute::ReadNone, Attribute::ReadOnly, Attribute::NoFree},
- IgnoreSubsumingPositions, Attribute::NoFree);
- }
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.isFunctionScope() &&
- !IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return IRAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// Return true if "nofree" is assumed.
- bool isAssumedNoFree() const { return getAssumed(); }
-
- /// Return true if "nofree" is known.
- bool isKnownNoFree() const { return getKnown(); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANoFree &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANoFree"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AANoFree
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An AbstractAttribute for noreturn.
-struct AANoReturn
- : public IRAttribute<Attribute::NoReturn,
- StateWrapper<BooleanState, AbstractAttribute>,
- AANoReturn> {
- AANoReturn(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// Return true if the underlying object is assumed to never return.
- bool isAssumedNoReturn() const { return getAssumed(); }
-
- /// Return true if the underlying object is known to never return.
- bool isKnownNoReturn() const { return getKnown(); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANoReturn &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANoReturn"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AANoReturn
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for liveness abstract attribute.
-struct AAIsDead
- : public StateWrapper<BitIntegerState<uint8_t, 3, 0>, AbstractAttribute> {
- using Base = StateWrapper<BitIntegerState<uint8_t, 3, 0>, AbstractAttribute>;
- AAIsDead(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
- return isa<Function>(IRP.getAnchorValue()) &&
- !cast<Function>(IRP.getAnchorValue()).isDeclaration();
- return true;
- }
-
- /// State encoding bits. A set bit in the state means the property holds.
- enum {
- HAS_NO_EFFECT = 1 << 0,
- IS_REMOVABLE = 1 << 1,
-
- IS_DEAD = HAS_NO_EFFECT | IS_REMOVABLE,
- };
- static_assert(IS_DEAD == getBestState(), "Unexpected BEST_STATE value");
-
-protected:
- /// The query functions are protected such that other attributes need to go
- /// through the Attributor interfaces: `Attributor::isAssumedDead(...)`
-
- /// Returns true if the underlying value is assumed dead.
- virtual bool isAssumedDead() const = 0;
-
- /// Returns true if the underlying value is known dead.
- virtual bool isKnownDead() const = 0;
-
- /// Returns true if \p BB is known dead.
- virtual bool isKnownDead(const BasicBlock *BB) const = 0;
-
- /// Returns true if \p I is assumed dead.
- virtual bool isAssumedDead(const Instruction *I) const = 0;
-
- /// Returns true if \p I is known dead.
- virtual bool isKnownDead(const Instruction *I) const = 0;
-
- /// Return true if the underlying value is a store that is known to be
- /// removable. This is different from dead stores as the removable store
- /// can have an effect on live values, especially loads, but that effect
- /// is propagated which allows us to remove the store in turn.
- virtual bool isRemovableStore() const { return false; }
-
- /// This method is used to check if at least one instruction in a collection
- /// of instructions is live.
- template <typename T> bool isLiveInstSet(T begin, T end) const {
- for (const auto &I : llvm::make_range(begin, end)) {
- assert(I->getFunction() == getIRPosition().getAssociatedFunction() &&
- "Instruction must be in the same anchor scope function.");
-
- if (!isAssumedDead(I))
- return true;
- }
-
- return false;
- }
-
-public:
- /// Create an abstract attribute view for the position \p IRP.
- static AAIsDead &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// Determine if \p F might catch asynchronous exceptions.
- static bool mayCatchAsynchronousExceptions(const Function &F) {
- return F.hasPersonalityFn() && !canSimplifyInvokeNoUnwind(&F);
- }
-
- /// Returns true if \p BB is assumed dead.
- virtual bool isAssumedDead(const BasicBlock *BB) const = 0;
-
- /// Return if the edge from \p From BB to \p To BB is assumed dead.
- /// This is specifically useful in AAReachability.
- virtual bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const {
- return false;
- }
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAIsDead"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AAIsDead
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-
- friend struct Attributor;
-};
-
-/// State for dereferenceable attribute
-struct DerefState : AbstractState {
-
- static DerefState getBestState() { return DerefState(); }
- static DerefState getBestState(const DerefState &) { return getBestState(); }
-
- /// Return the worst possible representable state.
- static DerefState getWorstState() {
- DerefState DS;
- DS.indicatePessimisticFixpoint();
- return DS;
- }
- static DerefState getWorstState(const DerefState &) {
- return getWorstState();
- }
-
- /// State representing for dereferenceable bytes.
- IncIntegerState<> DerefBytesState;
-
- /// Map representing for accessed memory offsets and sizes.
- /// A key is Offset and a value is size.
- /// If there is a load/store instruction something like,
- /// p[offset] = v;
- /// (offset, sizeof(v)) will be inserted to this map.
- /// std::map is used because we want to iterate keys in ascending order.
- std::map<int64_t, uint64_t> AccessedBytesMap;
-
- /// Helper function to calculate dereferenceable bytes from current known
- /// bytes and accessed bytes.
- ///
- /// int f(int *A){
- /// *A = 0;
- /// *(A+2) = 2;
- /// *(A+1) = 1;
- /// *(A+10) = 10;
- /// }
- /// ```
- /// In that case, AccessedBytesMap is `{0:4, 4:4, 8:4, 40:4}`.
- /// AccessedBytesMap is std::map so it is iterated in accending order on
- /// key(Offset). So KnownBytes will be updated like this:
- ///
- /// |Access | KnownBytes
- /// |(0, 4)| 0 -> 4
- /// |(4, 4)| 4 -> 8
- /// |(8, 4)| 8 -> 12
- /// |(40, 4) | 12 (break)
- void computeKnownDerefBytesFromAccessedMap() {
- int64_t KnownBytes = DerefBytesState.getKnown();
- for (auto &Access : AccessedBytesMap) {
- if (KnownBytes < Access.first)
- break;
- KnownBytes = std::max(KnownBytes, Access.first + (int64_t)Access.second);
- }
-
- DerefBytesState.takeKnownMaximum(KnownBytes);
- }
-
- /// State representing that whether the value is globaly dereferenceable.
- BooleanState GlobalState;
-
- /// See AbstractState::isValidState()
- bool isValidState() const override { return DerefBytesState.isValidState(); }
-
- /// See AbstractState::isAtFixpoint()
- bool isAtFixpoint() const override {
- return !isValidState() ||
- (DerefBytesState.isAtFixpoint() && GlobalState.isAtFixpoint());
- }
-
- /// See AbstractState::indicateOptimisticFixpoint(...)
- ChangeStatus indicateOptimisticFixpoint() override {
- DerefBytesState.indicateOptimisticFixpoint();
- GlobalState.indicateOptimisticFixpoint();
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractState::indicatePessimisticFixpoint(...)
- ChangeStatus indicatePessimisticFixpoint() override {
- DerefBytesState.indicatePessimisticFixpoint();
- GlobalState.indicatePessimisticFixpoint();
- return ChangeStatus::CHANGED;
- }
-
- /// Update known dereferenceable bytes.
- void takeKnownDerefBytesMaximum(uint64_t Bytes) {
- DerefBytesState.takeKnownMaximum(Bytes);
-
- // Known bytes might increase.
- computeKnownDerefBytesFromAccessedMap();
- }
-
- /// Update assumed dereferenceable bytes.
- void takeAssumedDerefBytesMinimum(uint64_t Bytes) {
- DerefBytesState.takeAssumedMinimum(Bytes);
- }
-
- /// Add accessed bytes to the map.
- void addAccessedBytes(int64_t Offset, uint64_t Size) {
- uint64_t &AccessedBytes = AccessedBytesMap[Offset];
- AccessedBytes = std::max(AccessedBytes, Size);
-
- // Known bytes might increase.
- computeKnownDerefBytesFromAccessedMap();
- }
-
- /// Equality for DerefState.
- bool operator==(const DerefState &R) const {
- return this->DerefBytesState == R.DerefBytesState &&
- this->GlobalState == R.GlobalState;
- }
-
- /// Inequality for DerefState.
- bool operator!=(const DerefState &R) const { return !(*this == R); }
-
- /// See IntegerStateBase::operator^=
- DerefState operator^=(const DerefState &R) {
- DerefBytesState ^= R.DerefBytesState;
- GlobalState ^= R.GlobalState;
- return *this;
- }
-
- /// See IntegerStateBase::operator+=
- DerefState operator+=(const DerefState &R) {
- DerefBytesState += R.DerefBytesState;
- GlobalState += R.GlobalState;
- return *this;
- }
-
- /// See IntegerStateBase::operator&=
- DerefState operator&=(const DerefState &R) {
- DerefBytesState &= R.DerefBytesState;
- GlobalState &= R.GlobalState;
- return *this;
- }
-
- /// See IntegerStateBase::operator|=
- DerefState operator|=(const DerefState &R) {
- DerefBytesState |= R.DerefBytesState;
- GlobalState |= R.GlobalState;
- return *this;
- }
-};
-
-/// An abstract interface for all dereferenceable attribute.
-struct AADereferenceable
- : public IRAttribute<Attribute::Dereferenceable,
- StateWrapper<DerefState, AbstractAttribute>,
- AADereferenceable> {
- AADereferenceable(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return IRAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// Return true if we assume that underlying value is
- /// dereferenceable(_or_null) globally.
- bool isAssumedGlobal() const { return GlobalState.getAssumed(); }
-
- /// Return true if we know that underlying value is
- /// dereferenceable(_or_null) globally.
- bool isKnownGlobal() const { return GlobalState.getKnown(); }
-
- /// Return assumed dereferenceable bytes.
- uint32_t getAssumedDereferenceableBytes() const {
- return DerefBytesState.getAssumed();
- }
-
- /// Return known dereferenceable bytes.
- uint32_t getKnownDereferenceableBytes() const {
- return DerefBytesState.getKnown();
- }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AADereferenceable &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AADereferenceable"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AADereferenceable
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-using AAAlignmentStateType =
- IncIntegerState<uint64_t, Value::MaximumAlignment, 1>;
-/// An abstract interface for all align attributes.
-struct AAAlign
- : public IRAttribute<Attribute::Alignment,
- StateWrapper<AAAlignmentStateType, AbstractAttribute>,
- AAAlign> {
- AAAlign(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return IRAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// Return assumed alignment.
- Align getAssumedAlign() const { return Align(getAssumed()); }
-
- /// Return known alignment.
- Align getKnownAlign() const { return Align(getKnown()); }
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAAlign"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AAAlign
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAAlign &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface to track if a value leaves it's defining function
-/// instance.
-/// TODO: We should make it a ternary AA tracking uniqueness, and uniqueness
-/// wrt. the Attributor analysis separately.
-struct AAInstanceInfo : public StateWrapper<BooleanState, AbstractAttribute> {
- AAInstanceInfo(const IRPosition &IRP, Attributor &A)
- : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
-
- /// Return true if we know that the underlying value is unique in its scope
- /// wrt. the Attributor analysis. That means it might not be unique but we can
- /// still use pointer equality without risking to represent two instances with
- /// one `llvm::Value`.
- bool isKnownUniqueForAnalysis() const { return isKnown(); }
-
- /// Return true if we assume that the underlying value is unique in its scope
- /// wrt. the Attributor analysis. That means it might not be unique but we can
- /// still use pointer equality without risking to represent two instances with
- /// one `llvm::Value`.
- bool isAssumedUniqueForAnalysis() const { return isAssumed(); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAInstanceInfo &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAInstanceInfo"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAInstanceInfo
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for all nocapture attributes.
-struct AANoCapture
- : public IRAttribute<
- Attribute::NoCapture,
- StateWrapper<BitIntegerState<uint16_t, 7, 0>, AbstractAttribute>,
- AANoCapture> {
- AANoCapture(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// See IRAttribute::isImpliedByIR
- static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions = false);
-
- /// Update \p State according to the capture capabilities of \p F for position
- /// \p IRP.
- static void determineFunctionCaptureCapabilities(const IRPosition &IRP,
- const Function &F,
- BitIntegerState &State);
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return IRAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// State encoding bits. A set bit in the state means the property holds.
- /// NO_CAPTURE is the best possible state, 0 the worst possible state.
- enum {
- NOT_CAPTURED_IN_MEM = 1 << 0,
- NOT_CAPTURED_IN_INT = 1 << 1,
- NOT_CAPTURED_IN_RET = 1 << 2,
-
- /// If we do not capture the value in memory or through integers we can only
- /// communicate it back as a derived pointer.
- NO_CAPTURE_MAYBE_RETURNED = NOT_CAPTURED_IN_MEM | NOT_CAPTURED_IN_INT,
-
- /// If we do not capture the value in memory, through integers, or as a
- /// derived pointer we know it is not captured.
- NO_CAPTURE =
- NOT_CAPTURED_IN_MEM | NOT_CAPTURED_IN_INT | NOT_CAPTURED_IN_RET,
- };
-
- /// Return true if we know that the underlying value is not captured in its
- /// respective scope.
- bool isKnownNoCapture() const { return isKnown(NO_CAPTURE); }
-
- /// Return true if we assume that the underlying value is not captured in its
- /// respective scope.
- bool isAssumedNoCapture() const { return isAssumed(NO_CAPTURE); }
-
- /// Return true if we know that the underlying value is not captured in its
- /// respective scope but we allow it to escape through a "return".
- bool isKnownNoCaptureMaybeReturned() const {
- return isKnown(NO_CAPTURE_MAYBE_RETURNED);
- }
-
- /// Return true if we assume that the underlying value is not captured in its
- /// respective scope but we allow it to escape through a "return".
- bool isAssumedNoCaptureMaybeReturned() const {
- return isAssumed(NO_CAPTURE_MAYBE_RETURNED);
- }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANoCapture &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANoCapture"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AANoCapture
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-struct ValueSimplifyStateType : public AbstractState {
-
- ValueSimplifyStateType(Type *Ty) : Ty(Ty) {}
-
- static ValueSimplifyStateType getBestState(Type *Ty) {
- return ValueSimplifyStateType(Ty);
- }
- static ValueSimplifyStateType getBestState(const ValueSimplifyStateType &VS) {
- return getBestState(VS.Ty);
- }
-
- /// Return the worst possible representable state.
- static ValueSimplifyStateType getWorstState(Type *Ty) {
- ValueSimplifyStateType DS(Ty);
- DS.indicatePessimisticFixpoint();
- return DS;
- }
- static ValueSimplifyStateType
- getWorstState(const ValueSimplifyStateType &VS) {
- return getWorstState(VS.Ty);
- }
-
- /// See AbstractState::isValidState(...)
- bool isValidState() const override { return BS.isValidState(); }
-
- /// See AbstractState::isAtFixpoint(...)
- bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
-
- /// Return the assumed state encoding.
- ValueSimplifyStateType getAssumed() { return *this; }
- const ValueSimplifyStateType &getAssumed() const { return *this; }
-
- /// See AbstractState::indicatePessimisticFixpoint(...)
- ChangeStatus indicatePessimisticFixpoint() override {
- return BS.indicatePessimisticFixpoint();
- }
-
- /// See AbstractState::indicateOptimisticFixpoint(...)
- ChangeStatus indicateOptimisticFixpoint() override {
- return BS.indicateOptimisticFixpoint();
- }
-
- /// "Clamp" this state with \p PVS.
- ValueSimplifyStateType operator^=(const ValueSimplifyStateType &VS) {
- BS ^= VS.BS;
- unionAssumed(VS.SimplifiedAssociatedValue);
- return *this;
- }
-
- bool operator==(const ValueSimplifyStateType &RHS) const {
- if (isValidState() != RHS.isValidState())
- return false;
- if (!isValidState() && !RHS.isValidState())
- return true;
- return SimplifiedAssociatedValue == RHS.SimplifiedAssociatedValue;
- }
-
-protected:
- /// The type of the original value.
- Type *Ty;
-
- /// Merge \p Other into the currently assumed simplified value
- bool unionAssumed(std::optional<Value *> Other);
-
- /// Helper to track validity and fixpoint
- BooleanState BS;
-
- /// An assumed simplified value. Initially, it is set to std::nullopt, which
- /// means that the value is not clear under current assumption. If in the
- /// pessimistic state, getAssumedSimplifiedValue doesn't return this value but
- /// returns orignal associated value.
- std::optional<Value *> SimplifiedAssociatedValue;
-};
-
-/// An abstract interface for value simplify abstract attribute.
-struct AAValueSimplify
- : public StateWrapper<ValueSimplifyStateType, AbstractAttribute, Type *> {
- using Base = StateWrapper<ValueSimplifyStateType, AbstractAttribute, Type *>;
- AAValueSimplify(const IRPosition &IRP, Attributor &A)
- : Base(IRP, IRP.getAssociatedType()) {}
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAValueSimplify &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAValueSimplify"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAValueSimplify
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-
-private:
- /// Return an assumed simplified value if a single candidate is found. If
- /// there cannot be one, return original value. If it is not clear yet, return
- /// std::nullopt.
- ///
- /// Use `Attributor::getAssumedSimplified` for value simplification.
- virtual std::optional<Value *>
- getAssumedSimplifiedValue(Attributor &A) const = 0;
-
- friend struct Attributor;
-};
-
-struct AAHeapToStack : public StateWrapper<BooleanState, AbstractAttribute> {
- using Base = StateWrapper<BooleanState, AbstractAttribute>;
- AAHeapToStack(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// Returns true if HeapToStack conversion is assumed to be possible.
- virtual bool isAssumedHeapToStack(const CallBase &CB) const = 0;
-
- /// Returns true if HeapToStack conversion is assumed and the CB is a
- /// callsite to a free operation to be removed.
- virtual bool isAssumedHeapToStackRemovedFree(CallBase &CB) const = 0;
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAHeapToStack &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAHeapToStack"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AAHeapToStack
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for privatizability.
-///
-/// A pointer is privatizable if it can be replaced by a new, private one.
-/// Privatizing pointer reduces the use count, interaction between unrelated
-/// code parts.
-///
-/// In order for a pointer to be privatizable its value cannot be observed
-/// (=nocapture), it is (for now) not written (=readonly & noalias), we know
-/// what values are necessary to make the private copy look like the original
-/// one, and the values we need can be loaded (=dereferenceable).
-struct AAPrivatizablePtr
- : public StateWrapper<BooleanState, AbstractAttribute> {
- using Base = StateWrapper<BooleanState, AbstractAttribute>;
- AAPrivatizablePtr(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return AbstractAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// Returns true if pointer privatization is assumed to be possible.
- bool isAssumedPrivatizablePtr() const { return getAssumed(); }
-
- /// Returns true if pointer privatization is known to be possible.
- bool isKnownPrivatizablePtr() const { return getKnown(); }
-
- /// See AbstractAttribute::requiresCallersForArgOrFunction
- static bool requiresCallersForArgOrFunction() { return true; }
-
- /// Return the type we can choose for a private copy of the underlying
- /// value. std::nullopt means it is not clear yet, nullptr means there is
- /// none.
- virtual std::optional<Type *> getPrivatizableType() const = 0;
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAPrivatizablePtr &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAPrivatizablePtr"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAPricatizablePtr
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for memory access kind related attributes
-/// (readnone/readonly/writeonly).
-struct AAMemoryBehavior
- : public IRAttribute<
- Attribute::None,
- StateWrapper<BitIntegerState<uint8_t, 3>, AbstractAttribute>,
- AAMemoryBehavior> {
- AAMemoryBehavior(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// See AbstractAttribute::hasTrivialInitializer.
- static bool hasTrivialInitializer() { return false; }
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.isFunctionScope() &&
- !IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return IRAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// State encoding bits. A set bit in the state means the property holds.
- /// BEST_STATE is the best possible state, 0 the worst possible state.
- enum {
- NO_READS = 1 << 0,
- NO_WRITES = 1 << 1,
- NO_ACCESSES = NO_READS | NO_WRITES,
-
- BEST_STATE = NO_ACCESSES,
- };
- static_assert(BEST_STATE == getBestState(), "Unexpected BEST_STATE value");
-
- /// Return true if we know that the underlying value is not read or accessed
- /// in its respective scope.
- bool isKnownReadNone() const { return isKnown(NO_ACCESSES); }
-
- /// Return true if we assume that the underlying value is not read or accessed
- /// in its respective scope.
- bool isAssumedReadNone() const { return isAssumed(NO_ACCESSES); }
-
- /// Return true if we know that the underlying value is not accessed
- /// (=written) in its respective scope.
- bool isKnownReadOnly() const { return isKnown(NO_WRITES); }
-
- /// Return true if we assume that the underlying value is not accessed
- /// (=written) in its respective scope.
- bool isAssumedReadOnly() const { return isAssumed(NO_WRITES); }
-
- /// Return true if we know that the underlying value is not read in its
- /// respective scope.
- bool isKnownWriteOnly() const { return isKnown(NO_READS); }
-
- /// Return true if we assume that the underlying value is not read in its
- /// respective scope.
- bool isAssumedWriteOnly() const { return isAssumed(NO_READS); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAMemoryBehavior &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAMemoryBehavior"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAMemoryBehavior
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for all memory location attributes
-/// (readnone/argmemonly/inaccessiblememonly/inaccessibleorargmemonly).
-struct AAMemoryLocation
- : public IRAttribute<
- Attribute::None,
- StateWrapper<BitIntegerState<uint32_t, 511>, AbstractAttribute>,
- AAMemoryLocation> {
- using MemoryLocationsKind = StateType::base_t;
-
- AAMemoryLocation(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// See AbstractAttribute::requiresCalleeForCallBase.
- static bool requiresCalleeForCallBase() { return true; }
-
- /// See AbstractAttribute::hasTrivialInitializer.
- static bool hasTrivialInitializer() { return false; }
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.isFunctionScope() &&
- !IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return IRAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// Encoding of different locations that could be accessed by a memory
- /// access.
- enum {
- ALL_LOCATIONS = 0,
- NO_LOCAL_MEM = 1 << 0,
- NO_CONST_MEM = 1 << 1,
- NO_GLOBAL_INTERNAL_MEM = 1 << 2,
- NO_GLOBAL_EXTERNAL_MEM = 1 << 3,
- NO_GLOBAL_MEM = NO_GLOBAL_INTERNAL_MEM | NO_GLOBAL_EXTERNAL_MEM,
- NO_ARGUMENT_MEM = 1 << 4,
- NO_INACCESSIBLE_MEM = 1 << 5,
- NO_MALLOCED_MEM = 1 << 6,
- NO_UNKOWN_MEM = 1 << 7,
- NO_LOCATIONS = NO_LOCAL_MEM | NO_CONST_MEM | NO_GLOBAL_INTERNAL_MEM |
- NO_GLOBAL_EXTERNAL_MEM | NO_ARGUMENT_MEM |
- NO_INACCESSIBLE_MEM | NO_MALLOCED_MEM | NO_UNKOWN_MEM,
-
- // Helper bit to track if we gave up or not.
- VALID_STATE = NO_LOCATIONS + 1,
-
- BEST_STATE = NO_LOCATIONS | VALID_STATE,
- };
- static_assert(BEST_STATE == getBestState(), "Unexpected BEST_STATE value");
-
- /// Return true if we know that the associated functions has no observable
- /// accesses.
- bool isKnownReadNone() const { return isKnown(NO_LOCATIONS); }
-
- /// Return true if we assume that the associated functions has no observable
- /// accesses.
- bool isAssumedReadNone() const {
- return isAssumed(NO_LOCATIONS) || isAssumedStackOnly();
- }
-
- /// Return true if we know that the associated functions has at most
- /// local/stack accesses.
- bool isKnowStackOnly() const {
- return isKnown(inverseLocation(NO_LOCAL_MEM, true, true));
- }
-
- /// Return true if we assume that the associated functions has at most
- /// local/stack accesses.
- bool isAssumedStackOnly() const {
- return isAssumed(inverseLocation(NO_LOCAL_MEM, true, true));
- }
-
- /// Return true if we know that the underlying value will only access
- /// inaccesible memory only (see Attribute::InaccessibleMemOnly).
- bool isKnownInaccessibleMemOnly() const {
- return isKnown(inverseLocation(NO_INACCESSIBLE_MEM, true, true));
- }
-
- /// Return true if we assume that the underlying value will only access
- /// inaccesible memory only (see Attribute::InaccessibleMemOnly).
- bool isAssumedInaccessibleMemOnly() const {
- return isAssumed(inverseLocation(NO_INACCESSIBLE_MEM, true, true));
- }
-
- /// Return true if we know that the underlying value will only access
- /// argument pointees (see Attribute::ArgMemOnly).
- bool isKnownArgMemOnly() const {
- return isKnown(inverseLocation(NO_ARGUMENT_MEM, true, true));
- }
-
- /// Return true if we assume that the underlying value will only access
- /// argument pointees (see Attribute::ArgMemOnly).
- bool isAssumedArgMemOnly() const {
- return isAssumed(inverseLocation(NO_ARGUMENT_MEM, true, true));
- }
-
- /// Return true if we know that the underlying value will only access
- /// inaccesible memory or argument pointees (see
- /// Attribute::InaccessibleOrArgMemOnly).
- bool isKnownInaccessibleOrArgMemOnly() const {
- return isKnown(
- inverseLocation(NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, true, true));
- }
-
- /// Return true if we assume that the underlying value will only access
- /// inaccesible memory or argument pointees (see
- /// Attribute::InaccessibleOrArgMemOnly).
- bool isAssumedInaccessibleOrArgMemOnly() const {
- return isAssumed(
- inverseLocation(NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, true, true));
- }
-
- /// Return true if the underlying value may access memory through arguement
- /// pointers of the associated function, if any.
- bool mayAccessArgMem() const { return !isAssumed(NO_ARGUMENT_MEM); }
-
- /// Return true if only the memory locations specififed by \p MLK are assumed
- /// to be accessed by the associated function.
- bool isAssumedSpecifiedMemOnly(MemoryLocationsKind MLK) const {
- return isAssumed(MLK);
- }
-
- /// Return the locations that are assumed to be not accessed by the associated
- /// function, if any.
- MemoryLocationsKind getAssumedNotAccessedLocation() const {
- return getAssumed();
- }
-
- /// Return the inverse of location \p Loc, thus for NO_XXX the return
- /// describes ONLY_XXX. The flags \p AndLocalMem and \p AndConstMem determine
- /// if local (=stack) and constant memory are allowed as well. Most of the
- /// time we do want them to be included, e.g., argmemonly allows accesses via
- /// argument pointers or local or constant memory accesses.
- static MemoryLocationsKind
- inverseLocation(MemoryLocationsKind Loc, bool AndLocalMem, bool AndConstMem) {
- return NO_LOCATIONS & ~(Loc | (AndLocalMem ? NO_LOCAL_MEM : 0) |
- (AndConstMem ? NO_CONST_MEM : 0));
- };
-
- /// Return the locations encoded by \p MLK as a readable string.
- static std::string getMemoryLocationsAsStr(MemoryLocationsKind MLK);
-
- /// Simple enum to distinguish read/write/read-write accesses.
- enum AccessKind {
- NONE = 0,
- READ = 1 << 0,
- WRITE = 1 << 1,
- READ_WRITE = READ | WRITE,
- };
-
- /// Check \p Pred on all accesses to the memory kinds specified by \p MLK.
- ///
- /// This method will evaluate \p Pred on all accesses (access instruction +
- /// underlying accessed memory pointer) and it will return true if \p Pred
- /// holds every time.
- virtual bool checkForAllAccessesToMemoryKind(
- function_ref<bool(const Instruction *, const Value *, AccessKind,
- MemoryLocationsKind)>
- Pred,
- MemoryLocationsKind MLK) const = 0;
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAMemoryLocation &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractState::getAsStr(Attributor).
- const std::string getAsStr(Attributor *A) const override {
- return getMemoryLocationsAsStr(getAssumedNotAccessedLocation());
- }
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAMemoryLocation"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAMemoryLocation
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for range value analysis.
-struct AAValueConstantRange
- : public StateWrapper<IntegerRangeState, AbstractAttribute, uint32_t> {
- using Base = StateWrapper<IntegerRangeState, AbstractAttribute, uint32_t>;
- AAValueConstantRange(const IRPosition &IRP, Attributor &A)
- : Base(IRP, IRP.getAssociatedType()->getIntegerBitWidth()) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isIntegerTy())
- return false;
- return AbstractAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// See AbstractAttribute::requiresCallersForArgOrFunction
- static bool requiresCallersForArgOrFunction() { return true; }
-
- /// See AbstractAttribute::getState(...).
- IntegerRangeState &getState() override { return *this; }
- const IntegerRangeState &getState() const override { return *this; }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAValueConstantRange &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// Return an assumed range for the associated value a program point \p CtxI.
- /// If \p I is nullptr, simply return an assumed range.
- virtual ConstantRange
- getAssumedConstantRange(Attributor &A,
- const Instruction *CtxI = nullptr) const = 0;
-
- /// Return a known range for the associated value at a program point \p CtxI.
- /// If \p I is nullptr, simply return a known range.
- virtual ConstantRange
- getKnownConstantRange(Attributor &A,
- const Instruction *CtxI = nullptr) const = 0;
-
- /// Return an assumed constant for the associated value a program point \p
- /// CtxI.
- std::optional<Constant *>
- getAssumedConstant(Attributor &A, const Instruction *CtxI = nullptr) const {
- ConstantRange RangeV = getAssumedConstantRange(A, CtxI);
- if (auto *C = RangeV.getSingleElement()) {
- Type *Ty = getAssociatedValue().getType();
- return cast_or_null<Constant>(
- AA::getWithType(*ConstantInt::get(Ty->getContext(), *C), *Ty));
- }
- if (RangeV.isEmptySet())
- return std::nullopt;
- return nullptr;
- }
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAValueConstantRange"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAValueConstantRange
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// A class for a set state.
-/// The assumed boolean state indicates whether the corresponding set is full
-/// set or not. If the assumed state is false, this is the worst state. The
-/// worst state (invalid state) of set of potential values is when the set
-/// contains every possible value (i.e. we cannot in any way limit the value
-/// that the target position can take). That never happens naturally, we only
-/// force it. As for the conditions under which we force it, see
-/// AAPotentialConstantValues.
-template <typename MemberTy> struct PotentialValuesState : AbstractState {
- using SetTy = SmallSetVector<MemberTy, 8>;
-
- PotentialValuesState() : IsValidState(true), UndefIsContained(false) {}
-
- PotentialValuesState(bool IsValid)
- : IsValidState(IsValid), UndefIsContained(false) {}
-
- /// See AbstractState::isValidState(...)
- bool isValidState() const override { return IsValidState.isValidState(); }
-
- /// See AbstractState::isAtFixpoint(...)
- bool isAtFixpoint() const override { return IsValidState.isAtFixpoint(); }
-
- /// See AbstractState::indicatePessimisticFixpoint(...)
- ChangeStatus indicatePessimisticFixpoint() override {
- return IsValidState.indicatePessimisticFixpoint();
- }
-
- /// See AbstractState::indicateOptimisticFixpoint(...)
- ChangeStatus indicateOptimisticFixpoint() override {
- return IsValidState.indicateOptimisticFixpoint();
- }
-
- /// Return the assumed state
- PotentialValuesState &getAssumed() { return *this; }
- const PotentialValuesState &getAssumed() const { return *this; }
-
- /// Return this set. We should check whether this set is valid or not by
- /// isValidState() before calling this function.
- const SetTy &getAssumedSet() const {
- assert(isValidState() && "This set shoud not be used when it is invalid!");
- return Set;
- }
-
- /// Returns whether this state contains an undef value or not.
- bool undefIsContained() const {
- assert(isValidState() && "This flag shoud not be used when it is invalid!");
- return UndefIsContained;
- }
-
- bool operator==(const PotentialValuesState &RHS) const {
- if (isValidState() != RHS.isValidState())
- return false;
- if (!isValidState() && !RHS.isValidState())
- return true;
- if (undefIsContained() != RHS.undefIsContained())
- return false;
- return Set == RHS.getAssumedSet();
- }
-
- /// Maximum number of potential values to be tracked.
- /// This is set by -attributor-max-potential-values command line option
- static unsigned MaxPotentialValues;
-
- /// Return empty set as the best state of potential values.
- static PotentialValuesState getBestState() {
- return PotentialValuesState(true);
- }
-
- static PotentialValuesState getBestState(const PotentialValuesState &PVS) {
- return getBestState();
- }
-
- /// Return full set as the worst state of potential values.
- static PotentialValuesState getWorstState() {
- return PotentialValuesState(false);
- }
-
- /// Union assumed set with the passed value.
- void unionAssumed(const MemberTy &C) { insert(C); }
-
- /// Union assumed set with assumed set of the passed state \p PVS.
- void unionAssumed(const PotentialValuesState &PVS) { unionWith(PVS); }
-
- /// Union assumed set with an undef value.
- void unionAssumedWithUndef() { unionWithUndef(); }
-
- /// "Clamp" this state with \p PVS.
- PotentialValuesState operator^=(const PotentialValuesState &PVS) {
- IsValidState ^= PVS.IsValidState;
- unionAssumed(PVS);
- return *this;
- }
-
- PotentialValuesState operator&=(const PotentialValuesState &PVS) {
- IsValidState &= PVS.IsValidState;
- unionAssumed(PVS);
- return *this;
- }
-
- bool contains(const MemberTy &V) const {
- return !isValidState() ? true : Set.contains(V);
- }
-
-protected:
- SetTy &getAssumedSet() {
- assert(isValidState() && "This set shoud not be used when it is invalid!");
- return Set;
- }
-
-private:
- /// Check the size of this set, and invalidate when the size is no
- /// less than \p MaxPotentialValues threshold.
- void checkAndInvalidate() {
- if (Set.size() >= MaxPotentialValues)
- indicatePessimisticFixpoint();
- else
- reduceUndefValue();
- }
-
- /// If this state contains both undef and not undef, we can reduce
- /// undef to the not undef value.
- void reduceUndefValue() { UndefIsContained = UndefIsContained & Set.empty(); }
-
- /// Insert an element into this set.
- void insert(const MemberTy &C) {
- if (!isValidState())
- return;
- Set.insert(C);
- checkAndInvalidate();
- }
-
- /// Take union with R.
- void unionWith(const PotentialValuesState &R) {
- /// If this is a full set, do nothing.
- if (!isValidState())
- return;
- /// If R is full set, change L to a full set.
- if (!R.isValidState()) {
- indicatePessimisticFixpoint();
- return;
- }
- for (const MemberTy &C : R.Set)
- Set.insert(C);
- UndefIsContained |= R.undefIsContained();
- checkAndInvalidate();
- }
-
- /// Take union with an undef value.
- void unionWithUndef() {
- UndefIsContained = true;
- reduceUndefValue();
- }
-
- /// Take intersection with R.
- void intersectWith(const PotentialValuesState &R) {
- /// If R is a full set, do nothing.
- if (!R.isValidState())
- return;
- /// If this is a full set, change this to R.
- if (!isValidState()) {
- *this = R;
- return;
- }
- SetTy IntersectSet;
- for (const MemberTy &C : Set) {
- if (R.Set.count(C))
- IntersectSet.insert(C);
- }
- Set = IntersectSet;
- UndefIsContained &= R.undefIsContained();
- reduceUndefValue();
- }
-
- /// A helper state which indicate whether this state is valid or not.
- BooleanState IsValidState;
-
- /// Container for potential values
- SetTy Set;
-
- /// Flag for undef value
- bool UndefIsContained;
-};
-
-struct DenormalFPMathState : public AbstractState {
- struct DenormalState {
- DenormalMode Mode = DenormalMode::getInvalid();
- DenormalMode ModeF32 = DenormalMode::getInvalid();
-
- bool operator==(const DenormalState Other) const {
- return Mode == Other.Mode && ModeF32 == Other.ModeF32;
- }
-
- bool operator!=(const DenormalState Other) const {
- return Mode != Other.Mode || ModeF32 != Other.ModeF32;
- }
-
- bool isValid() const {
- return Mode.isValid() && ModeF32.isValid();
- }
-
- static DenormalMode::DenormalModeKind
- unionDenormalKind(DenormalMode::DenormalModeKind Callee,
- DenormalMode::DenormalModeKind Caller) {
- if (Caller == Callee)
- return Caller;
- if (Callee == DenormalMode::Dynamic)
- return Caller;
- if (Caller == DenormalMode::Dynamic)
- return Callee;
- return DenormalMode::Invalid;
- }
-
- static DenormalMode unionAssumed(DenormalMode Callee, DenormalMode Caller) {
- return DenormalMode{unionDenormalKind(Callee.Output, Caller.Output),
- unionDenormalKind(Callee.Input, Caller.Input)};
- }
-
- DenormalState unionWith(DenormalState Caller) const {
- DenormalState Callee(*this);
- Callee.Mode = unionAssumed(Callee.Mode, Caller.Mode);
- Callee.ModeF32 = unionAssumed(Callee.ModeF32, Caller.ModeF32);
- return Callee;
- }
- };
-
- DenormalState Known;
-
- /// Explicitly track whether we've hit a fixed point.
- bool IsAtFixedpoint = false;
-
- DenormalFPMathState() = default;
-
- DenormalState getKnown() const { return Known; }
-
- // There's only really known or unknown, there's no speculatively assumable
- // state.
- DenormalState getAssumed() const { return Known; }
-
- bool isValidState() const override {
- return Known.isValid();
- }
-
- /// Return true if there are no dynamic components to the denormal mode worth
- /// specializing.
- bool isModeFixed() const {
- return Known.Mode.Input != DenormalMode::Dynamic &&
- Known.Mode.Output != DenormalMode::Dynamic &&
- Known.ModeF32.Input != DenormalMode::Dynamic &&
- Known.ModeF32.Output != DenormalMode::Dynamic;
- }
-
- bool isAtFixpoint() const override {
- return IsAtFixedpoint;
- }
-
- ChangeStatus indicateFixpoint() {
- bool Changed = !IsAtFixedpoint;
- IsAtFixedpoint = true;
- return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
- }
-
- ChangeStatus indicateOptimisticFixpoint() override {
- return indicateFixpoint();
- }
-
- ChangeStatus indicatePessimisticFixpoint() override {
- return indicateFixpoint();
- }
-
- DenormalFPMathState operator^=(const DenormalFPMathState &Caller) {
- Known = Known.unionWith(Caller.getKnown());
- return *this;
- }
-};
-
-using PotentialConstantIntValuesState = PotentialValuesState<APInt>;
-using PotentialLLVMValuesState =
- PotentialValuesState<std::pair<AA::ValueAndContext, AA::ValueScope>>;
-
-raw_ostream &operator<<(raw_ostream &OS,
- const PotentialConstantIntValuesState &R);
-raw_ostream &operator<<(raw_ostream &OS, const PotentialLLVMValuesState &R);
-
-/// An abstract interface for potential values analysis.
-///
-/// This AA collects potential values for each IR position.
-/// An assumed set of potential values is initialized with the empty set (the
-/// best state) and it will grow monotonically as we find more potential values
-/// for this position.
-/// The set might be forced to the worst state, that is, to contain every
-/// possible value for this position in 2 cases.
-/// 1. We surpassed the \p MaxPotentialValues threshold. This includes the
-/// case that this position is affected (e.g. because of an operation) by a
-/// Value that is in the worst state.
-/// 2. We tried to initialize on a Value that we cannot handle (e.g. an
-/// operator we do not currently handle).
-///
-/// For non constant integers see AAPotentialValues.
-struct AAPotentialConstantValues
- : public StateWrapper<PotentialConstantIntValuesState, AbstractAttribute> {
- using Base = StateWrapper<PotentialConstantIntValuesState, AbstractAttribute>;
- AAPotentialConstantValues(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isIntegerTy())
- return false;
- return AbstractAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// See AbstractAttribute::requiresCallersForArgOrFunction
- static bool requiresCallersForArgOrFunction() { return true; }
-
- /// See AbstractAttribute::getState(...).
- PotentialConstantIntValuesState &getState() override { return *this; }
- const PotentialConstantIntValuesState &getState() const override {
- return *this;
- }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAPotentialConstantValues &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// Return assumed constant for the associated value
- std::optional<Constant *>
- getAssumedConstant(Attributor &A, const Instruction *CtxI = nullptr) const {
- if (!isValidState())
- return nullptr;
- if (getAssumedSet().size() == 1) {
- Type *Ty = getAssociatedValue().getType();
- return cast_or_null<Constant>(AA::getWithType(
- *ConstantInt::get(Ty->getContext(), *(getAssumedSet().begin())),
- *Ty));
- }
- if (getAssumedSet().size() == 0) {
- if (undefIsContained())
- return UndefValue::get(getAssociatedValue().getType());
- return std::nullopt;
- }
-
- return nullptr;
- }
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override {
- return "AAPotentialConstantValues";
- }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAPotentialConstantValues
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-struct AAPotentialValues
- : public StateWrapper<PotentialLLVMValuesState, AbstractAttribute> {
- using Base = StateWrapper<PotentialLLVMValuesState, AbstractAttribute>;
- AAPotentialValues(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// See AbstractAttribute::requiresCallersForArgOrFunction
- static bool requiresCallersForArgOrFunction() { return true; }
-
- /// See AbstractAttribute::getState(...).
- PotentialLLVMValuesState &getState() override { return *this; }
- const PotentialLLVMValuesState &getState() const override { return *this; }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAPotentialValues &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// Extract the single value in \p Values if any.
- static Value *getSingleValue(Attributor &A, const AbstractAttribute &AA,
- const IRPosition &IRP,
- SmallVectorImpl<AA::ValueAndContext> &Values);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAPotentialValues"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAPotentialValues
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-
-private:
- virtual bool getAssumedSimplifiedValues(
- Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
- AA::ValueScope, bool RecurseForSelectAndPHI = false) const = 0;
-
- friend struct Attributor;
-};
-
-/// An abstract interface for all noundef attributes.
-struct AANoUndef
- : public IRAttribute<Attribute::NoUndef,
- StateWrapper<BooleanState, AbstractAttribute>,
- AANoUndef> {
- AANoUndef(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// See IRAttribute::isImpliedByUndef
- static bool isImpliedByUndef() { return false; }
-
- /// See IRAttribute::isImpliedByPoison
- static bool isImpliedByPoison() { return false; }
-
- /// See IRAttribute::isImpliedByIR
- static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions = false);
-
- /// Return true if we assume that the underlying value is noundef.
- bool isAssumedNoUndef() const { return getAssumed(); }
-
- /// Return true if we know that underlying value is noundef.
- bool isKnownNoUndef() const { return getKnown(); }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANoUndef &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANoUndef"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AANoUndef
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-struct AANoFPClass
- : public IRAttribute<
- Attribute::NoFPClass,
- StateWrapper<BitIntegerState<uint32_t, fcAllFlags, fcNone>,
- AbstractAttribute>,
- AANoFPClass> {
- using Base = StateWrapper<BitIntegerState<uint32_t, fcAllFlags, fcNone>,
- AbstractAttribute>;
-
- AANoFPClass(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- Type *Ty = IRP.getAssociatedType();
- do {
- if (Ty->isFPOrFPVectorTy())
- return IRAttribute::isValidIRPositionForInit(A, IRP);
- if (!Ty->isArrayTy())
- break;
- Ty = Ty->getArrayElementType();
- } while (true);
- return false;
- }
-
- /// Return true if we assume that the underlying value is nofpclass.
- FPClassTest getAssumedNoFPClass() const {
- return static_cast<FPClassTest>(getAssumed());
- }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANoFPClass &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANoFPClass"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AANoFPClass
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-struct AACallGraphNode;
-struct AACallEdges;
-
-/// An Iterator for call edges, creates AACallEdges attributes in a lazy way.
-/// This iterator becomes invalid if the underlying edge list changes.
-/// So This shouldn't outlive a iteration of Attributor.
-class AACallEdgeIterator
- : public iterator_adaptor_base<AACallEdgeIterator,
- SetVector<Function *>::iterator> {
- AACallEdgeIterator(Attributor &A, SetVector<Function *>::iterator Begin)
- : iterator_adaptor_base(Begin), A(A) {}
-
-public:
- AACallGraphNode *operator*() const;
-
-private:
- Attributor &A;
- friend AACallEdges;
- friend AttributorCallGraph;
-};
-
-struct AACallGraphNode {
- AACallGraphNode(Attributor &A) : A(A) {}
- virtual ~AACallGraphNode() = default;
-
- virtual AACallEdgeIterator optimisticEdgesBegin() const = 0;
- virtual AACallEdgeIterator optimisticEdgesEnd() const = 0;
-
- /// Iterator range for exploring the call graph.
- iterator_range<AACallEdgeIterator> optimisticEdgesRange() const {
- return iterator_range<AACallEdgeIterator>(optimisticEdgesBegin(),
- optimisticEdgesEnd());
- }
-
-protected:
- /// Reference to Attributor needed for GraphTraits implementation.
- Attributor &A;
-};
-
-/// An abstract state for querying live call edges.
-/// This interface uses the Attributor's optimistic liveness
-/// information to compute the edges that are alive.
-struct AACallEdges : public StateWrapper<BooleanState, AbstractAttribute>,
- AACallGraphNode {
- using Base = StateWrapper<BooleanState, AbstractAttribute>;
-
- AACallEdges(const IRPosition &IRP, Attributor &A)
- : Base(IRP), AACallGraphNode(A) {}
-
- /// See AbstractAttribute::requiresNonAsmForCallBase.
- static bool requiresNonAsmForCallBase() { return false; }
-
- /// Get the optimistic edges.
- virtual const SetVector<Function *> &getOptimisticEdges() const = 0;
-
- /// Is there any call with a unknown callee.
- virtual bool hasUnknownCallee() const = 0;
-
- /// Is there any call with a unknown callee, excluding any inline asm.
- virtual bool hasNonAsmUnknownCallee() const = 0;
-
- /// Iterator for exploring the call graph.
- AACallEdgeIterator optimisticEdgesBegin() const override {
- return AACallEdgeIterator(A, getOptimisticEdges().begin());
- }
-
- /// Iterator for exploring the call graph.
- AACallEdgeIterator optimisticEdgesEnd() const override {
- return AACallEdgeIterator(A, getOptimisticEdges().end());
- }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AACallEdges &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AACallEdges"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AACallEdges.
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-// Synthetic root node for the Attributor's internal call graph.
-struct AttributorCallGraph : public AACallGraphNode {
- AttributorCallGraph(Attributor &A) : AACallGraphNode(A) {}
- virtual ~AttributorCallGraph() = default;
-
- AACallEdgeIterator optimisticEdgesBegin() const override {
- return AACallEdgeIterator(A, A.Functions.begin());
- }
-
- AACallEdgeIterator optimisticEdgesEnd() const override {
- return AACallEdgeIterator(A, A.Functions.end());
- }
-
- /// Force populate the entire call graph.
- void populateAll() const {
- for (const AACallGraphNode *AA : optimisticEdgesRange()) {
- // Nothing else to do here.
- (void)AA;
- }
- }
-
- void print();
-};
-
-template <> struct GraphTraits<AACallGraphNode *> {
- using NodeRef = AACallGraphNode *;
- using ChildIteratorType = AACallEdgeIterator;
-
- static AACallEdgeIterator child_begin(AACallGraphNode *Node) {
- return Node->optimisticEdgesBegin();
- }
-
- static AACallEdgeIterator child_end(AACallGraphNode *Node) {
- return Node->optimisticEdgesEnd();
- }
-};
-
-template <>
-struct GraphTraits<AttributorCallGraph *>
- : public GraphTraits<AACallGraphNode *> {
- using nodes_iterator = AACallEdgeIterator;
-
- static AACallGraphNode *getEntryNode(AttributorCallGraph *G) {
- return static_cast<AACallGraphNode *>(G);
- }
-
- static AACallEdgeIterator nodes_begin(const AttributorCallGraph *G) {
- return G->optimisticEdgesBegin();
- }
-
- static AACallEdgeIterator nodes_end(const AttributorCallGraph *G) {
- return G->optimisticEdgesEnd();
- }
-};
-
-template <>
-struct DOTGraphTraits<AttributorCallGraph *> : public DefaultDOTGraphTraits {
- DOTGraphTraits(bool Simple = false) : DefaultDOTGraphTraits(Simple) {}
-
- std::string getNodeLabel(const AACallGraphNode *Node,
- const AttributorCallGraph *Graph) {
- const AACallEdges *AACE = static_cast<const AACallEdges *>(Node);
- return AACE->getAssociatedFunction()->getName().str();
- }
-
- static bool isNodeHidden(const AACallGraphNode *Node,
- const AttributorCallGraph *Graph) {
- // Hide the synth root.
- return static_cast<const AACallGraphNode *>(Graph) == Node;
- }
-};
-
-struct AAExecutionDomain
- : public StateWrapper<BooleanState, AbstractAttribute> {
- using Base = StateWrapper<BooleanState, AbstractAttribute>;
- AAExecutionDomain(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// Summary about the execution domain of a block or instruction.
- struct ExecutionDomainTy {
- using BarriersSetTy = SmallPtrSet<CallBase *, 2>;
- using AssumesSetTy = SmallPtrSet<AssumeInst *, 4>;
-
- void addAssumeInst(Attributor &A, AssumeInst &AI) {
- EncounteredAssumes.insert(&AI);
- }
-
- void addAlignedBarrier(Attributor &A, CallBase &CB) {
- AlignedBarriers.insert(&CB);
- }
-
- void clearAssumeInstAndAlignedBarriers() {
- EncounteredAssumes.clear();
- AlignedBarriers.clear();
- }
-
- bool IsExecutedByInitialThreadOnly = true;
- bool IsReachedFromAlignedBarrierOnly = true;
- bool IsReachingAlignedBarrierOnly = true;
- bool EncounteredNonLocalSideEffect = false;
- BarriersSetTy AlignedBarriers;
- AssumesSetTy EncounteredAssumes;
- };
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAExecutionDomain &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName().
- const std::string getName() const override { return "AAExecutionDomain"; }
-
- /// See AbstractAttribute::getIdAddr().
- const char *getIdAddr() const override { return &ID; }
-
- /// Check if an instruction is executed only by the initial thread.
- bool isExecutedByInitialThreadOnly(const Instruction &I) const {
- return isExecutedByInitialThreadOnly(*I.getParent());
- }
-
- /// Check if a basic block is executed only by the initial thread.
- virtual bool isExecutedByInitialThreadOnly(const BasicBlock &) const = 0;
-
- /// Check if the instruction \p I is executed in an aligned region, that is,
- /// the synchronizing effects before and after \p I are both aligned barriers.
- /// This effectively means all threads execute \p I together.
- virtual bool isExecutedInAlignedRegion(Attributor &A,
- const Instruction &I) const = 0;
-
- virtual ExecutionDomainTy getExecutionDomain(const BasicBlock &) const = 0;
- /// Return the execution domain with which the call \p CB is entered and the
- /// one with which it is left.
- virtual std::pair<ExecutionDomainTy, ExecutionDomainTy>
- getExecutionDomain(const CallBase &CB) const = 0;
- virtual ExecutionDomainTy getFunctionExecutionDomain() const = 0;
-
- /// Helper function to determine if \p FI is a no-op given the information
- /// about its execution from \p ExecDomainAA.
- virtual bool isNoOpFence(const FenceInst &FI) const = 0;
-
- /// This function should return true if the type of the \p AA is
- /// AAExecutionDomain.
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract Attribute for computing reachability between functions.
-struct AAInterFnReachability
- : public StateWrapper<BooleanState, AbstractAttribute> {
- using Base = StateWrapper<BooleanState, AbstractAttribute>;
-
- AAInterFnReachability(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// If the function represented by this possition can reach \p Fn.
- bool canReach(Attributor &A, const Function &Fn) const {
- Function *Scope = getAnchorScope();
- if (!Scope || Scope->isDeclaration())
- return true;
- return instructionCanReach(A, Scope->getEntryBlock().front(), Fn);
- }
-
- /// Can \p Inst reach \p Fn.
- /// See also AA::isPotentiallyReachable.
- virtual bool instructionCanReach(
- Attributor &A, const Instruction &Inst, const Function &Fn,
- const AA::InstExclusionSetTy *ExclusionSet = nullptr) const = 0;
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAInterFnReachability &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAInterFnReachability"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is AACallEdges.
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract Attribute for determining the necessity of the convergent
-/// attribute.
-struct AANonConvergent : public StateWrapper<BooleanState, AbstractAttribute> {
- using Base = StateWrapper<BooleanState, AbstractAttribute>;
-
- AANonConvergent(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// Create an abstract attribute view for the position \p IRP.
- static AANonConvergent &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// Return true if "non-convergent" is assumed.
- bool isAssumedNotConvergent() const { return getAssumed(); }
-
- /// Return true if "non-convergent" is known.
- bool isKnownNotConvergent() const { return getKnown(); }
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AANonConvergent"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AANonConvergent.
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for struct information.
-struct AAPointerInfo : public AbstractAttribute {
- AAPointerInfo(const IRPosition &IRP) : AbstractAttribute(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return AbstractAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- enum AccessKind {
- // First two bits to distinguish may and must accesses.
- AK_MUST = 1 << 0,
- AK_MAY = 1 << 1,
-
- // Then two bits for read and write. These are not exclusive.
- AK_R = 1 << 2,
- AK_W = 1 << 3,
- AK_RW = AK_R | AK_W,
-
- // One special case for assumptions about memory content. These
- // are neither reads nor writes. They are however always modeled
- // as read to avoid using them for write removal.
- AK_ASSUMPTION = (1 << 4) | AK_MUST,
-
- // Helper for easy access.
- AK_MAY_READ = AK_MAY | AK_R,
- AK_MAY_WRITE = AK_MAY | AK_W,
- AK_MAY_READ_WRITE = AK_MAY | AK_R | AK_W,
- AK_MUST_READ = AK_MUST | AK_R,
- AK_MUST_WRITE = AK_MUST | AK_W,
- AK_MUST_READ_WRITE = AK_MUST | AK_R | AK_W,
- };
-
- /// A container for a list of ranges.
- struct RangeList {
- // The set of ranges rarely contains more than one element, and is unlikely
- // to contain more than say four elements. So we find the middle-ground with
- // a sorted vector. This avoids hard-coding a rarely used number like "four"
- // into every instance of a SmallSet.
- using RangeTy = AA::RangeTy;
- using VecTy = SmallVector<RangeTy>;
- using iterator = VecTy::iterator;
- using const_iterator = VecTy::const_iterator;
- VecTy Ranges;
-
- RangeList(const RangeTy &R) { Ranges.push_back(R); }
- RangeList(ArrayRef<int64_t> Offsets, int64_t Size) {
- Ranges.reserve(Offsets.size());
- for (unsigned i = 0, e = Offsets.size(); i != e; ++i) {
- assert(((i + 1 == e) || Offsets[i] < Offsets[i + 1]) &&
- "Expected strictly ascending offsets.");
- Ranges.emplace_back(Offsets[i], Size);
- }
- }
- RangeList() = default;
-
- iterator begin() { return Ranges.begin(); }
- iterator end() { return Ranges.end(); }
- const_iterator begin() const { return Ranges.begin(); }
- const_iterator end() const { return Ranges.end(); }
-
- // Helpers required for std::set_difference
- using value_type = RangeTy;
- void push_back(const RangeTy &R) {
- assert((Ranges.empty() || RangeTy::OffsetLessThan(Ranges.back(), R)) &&
- "Ensure the last element is the greatest.");
- Ranges.push_back(R);
- }
-
- /// Copy ranges from \p L that are not in \p R, into \p D.
- static void set_difference(const RangeList &L, const RangeList &R,
- RangeList &D) {
- std::set_difference(L.begin(), L.end(), R.begin(), R.end(),
- std::back_inserter(D), RangeTy::OffsetLessThan);
- }
-
- unsigned size() const { return Ranges.size(); }
-
- bool operator==(const RangeList &OI) const { return Ranges == OI.Ranges; }
-
- /// Merge the ranges in \p RHS into the current ranges.
- /// - Merging a list of unknown ranges makes the current list unknown.
- /// - Ranges with the same offset are merged according to RangeTy::operator&
- /// \return true if the current RangeList changed.
- bool merge(const RangeList &RHS) {
- if (isUnknown())
- return false;
- if (RHS.isUnknown()) {
- setUnknown();
- return true;
- }
-
- if (Ranges.empty()) {
- Ranges = RHS.Ranges;
- return true;
- }
-
- bool Changed = false;
- auto LPos = Ranges.begin();
- for (auto &R : RHS.Ranges) {
- auto Result = insert(LPos, R);
- if (isUnknown())
- return true;
- LPos = Result.first;
- Changed |= Result.second;
- }
- return Changed;
- }
-
- /// Insert \p R at the given iterator \p Pos, and merge if necessary.
- ///
- /// This assumes that all ranges before \p Pos are OffsetLessThan \p R, and
- /// then maintains the sorted order for the suffix list.
- ///
- /// \return The place of insertion and true iff anything changed.
- std::pair<iterator, bool> insert(iterator Pos, const RangeTy &R) {
- if (isUnknown())
- return std::make_pair(Ranges.begin(), false);
- if (R.offsetOrSizeAreUnknown()) {
- return std::make_pair(setUnknown(), true);
- }
-
- // Maintain this as a sorted vector of unique entries.
- auto LB = std::lower_bound(Pos, Ranges.end(), R, RangeTy::OffsetLessThan);
- if (LB == Ranges.end() || LB->Offset != R.Offset)
- return std::make_pair(Ranges.insert(LB, R), true);
- bool Changed = *LB != R;
- *LB &= R;
- if (LB->offsetOrSizeAreUnknown())
- return std::make_pair(setUnknown(), true);
- return std::make_pair(LB, Changed);
- }
-
- /// Insert the given range \p R, maintaining sorted order.
- ///
- /// \return The place of insertion and true iff anything changed.
- std::pair<iterator, bool> insert(const RangeTy &R) {
- return insert(Ranges.begin(), R);
- }
-
- /// Add the increment \p Inc to the offset of every range.
- void addToAllOffsets(int64_t Inc) {
- assert(!isUnassigned() &&
- "Cannot increment if the offset is not yet computed!");
- if (isUnknown())
- return;
- for (auto &R : Ranges) {
- R.Offset += Inc;
- }
- }
-
- /// Return true iff there is exactly one range and it is known.
- bool isUnique() const {
- return Ranges.size() == 1 && !Ranges.front().offsetOrSizeAreUnknown();
- }
-
- /// Return the unique range, assuming it exists.
- const RangeTy &getUnique() const {
- assert(isUnique() && "No unique range to return!");
- return Ranges.front();
- }
-
- /// Return true iff the list contains an unknown range.
- bool isUnknown() const {
- if (isUnassigned())
- return false;
- if (Ranges.front().offsetOrSizeAreUnknown()) {
- assert(Ranges.size() == 1 && "Unknown is a singleton range.");
- return true;
- }
- return false;
- }
-
- /// Discard all ranges and insert a single unknown range.
- iterator setUnknown() {
- Ranges.clear();
- Ranges.push_back(RangeTy::getUnknown());
- return Ranges.begin();
- }
-
- /// Return true if no ranges have been inserted.
- bool isUnassigned() const { return Ranges.size() == 0; }
- };
-
- /// An access description.
- struct Access {
- Access(Instruction *I, int64_t Offset, int64_t Size,
- std::optional<Value *> Content, AccessKind Kind, Type *Ty)
- : LocalI(I), RemoteI(I), Content(Content), Ranges(Offset, Size),
- Kind(Kind), Ty(Ty) {
- verify();
- }
- Access(Instruction *LocalI, Instruction *RemoteI, const RangeList &Ranges,
- std::optional<Value *> Content, AccessKind K, Type *Ty)
- : LocalI(LocalI), RemoteI(RemoteI), Content(Content), Ranges(Ranges),
- Kind(K), Ty(Ty) {
- if (Ranges.size() > 1) {
- Kind = AccessKind(Kind | AK_MAY);
- Kind = AccessKind(Kind & ~AK_MUST);
- }
- verify();
- }
- Access(Instruction *LocalI, Instruction *RemoteI, int64_t Offset,
- int64_t Size, std::optional<Value *> Content, AccessKind Kind,
- Type *Ty)
- : LocalI(LocalI), RemoteI(RemoteI), Content(Content),
- Ranges(Offset, Size), Kind(Kind), Ty(Ty) {
- verify();
- }
- Access(const Access &Other) = default;
-
- Access &operator=(const Access &Other) = default;
- bool operator==(const Access &R) const {
- return LocalI == R.LocalI && RemoteI == R.RemoteI && Ranges == R.Ranges &&
- Content == R.Content && Kind == R.Kind;
- }
- bool operator!=(const Access &R) const { return !(*this == R); }
-
- Access &operator&=(const Access &R) {
- assert(RemoteI == R.RemoteI && "Expected same instruction!");
- assert(LocalI == R.LocalI && "Expected same instruction!");
-
- // Note that every Access object corresponds to a unique Value, and only
- // accesses to the same Value are merged. Hence we assume that all ranges
- // are the same size. If ranges can be different size, then the contents
- // must be dropped.
- Ranges.merge(R.Ranges);
- Content =
- AA::combineOptionalValuesInAAValueLatice(Content, R.Content, Ty);
-
- // Combine the access kind, which results in a bitwise union.
- // If there is more than one range, then this must be a MAY.
- // If we combine a may and a must access we clear the must bit.
- Kind = AccessKind(Kind | R.Kind);
- if ((Kind & AK_MAY) || Ranges.size() > 1) {
- Kind = AccessKind(Kind | AK_MAY);
- Kind = AccessKind(Kind & ~AK_MUST);
- }
- verify();
- return *this;
- }
-
- void verify() {
- assert(isMustAccess() + isMayAccess() == 1 &&
- "Expect must or may access, not both.");
- assert(isAssumption() + isWrite() <= 1 &&
- "Expect assumption access or write access, never both.");
- assert((isMayAccess() || Ranges.size() == 1) &&
- "Cannot be a must access if there are multiple ranges.");
- }
-
- /// Return the access kind.
- AccessKind getKind() const { return Kind; }
-
- /// Return true if this is a read access.
- bool isRead() const { return Kind & AK_R; }
-
- /// Return true if this is a write access.
- bool isWrite() const { return Kind & AK_W; }
-
- /// Return true if this is a write access.
- bool isWriteOrAssumption() const { return isWrite() || isAssumption(); }
-
- /// Return true if this is an assumption access.
- bool isAssumption() const { return Kind == AK_ASSUMPTION; }
-
- bool isMustAccess() const {
- bool MustAccess = Kind & AK_MUST;
- assert((!MustAccess || Ranges.size() < 2) &&
- "Cannot be a must access if there are multiple ranges.");
- return MustAccess;
- }
-
- bool isMayAccess() const {
- bool MayAccess = Kind & AK_MAY;
- assert((MayAccess || Ranges.size() < 2) &&
- "Cannot be a must access if there are multiple ranges.");
- return MayAccess;
- }
-
- /// Return the instruction that causes the access with respect to the local
- /// scope of the associated attribute.
- Instruction *getLocalInst() const { return LocalI; }
-
- /// Return the actual instruction that causes the access.
- Instruction *getRemoteInst() const { return RemoteI; }
-
- /// Return true if the value written is not known yet.
- bool isWrittenValueYetUndetermined() const { return !Content; }
-
- /// Return true if the value written cannot be determined at all.
- bool isWrittenValueUnknown() const {
- return Content.has_value() && !*Content;
- }
-
- /// Set the value written to nullptr, i.e., unknown.
- void setWrittenValueUnknown() { Content = nullptr; }
-
- /// Return the type associated with the access, if known.
- Type *getType() const { return Ty; }
-
- /// Return the value writen, if any.
- Value *getWrittenValue() const {
- assert(!isWrittenValueYetUndetermined() &&
- "Value needs to be determined before accessing it.");
- return *Content;
- }
-
- /// Return the written value which can be `llvm::null` if it is not yet
- /// determined.
- std::optional<Value *> getContent() const { return Content; }
-
- bool hasUniqueRange() const { return Ranges.isUnique(); }
- const AA::RangeTy &getUniqueRange() const { return Ranges.getUnique(); }
-
- /// Add a range accessed by this Access.
- ///
- /// If there are multiple ranges, then this is a "may access".
- void addRange(int64_t Offset, int64_t Size) {
- Ranges.insert({Offset, Size});
- if (!hasUniqueRange()) {
- Kind = AccessKind(Kind | AK_MAY);
- Kind = AccessKind(Kind & ~AK_MUST);
- }
- }
-
- const RangeList &getRanges() const { return Ranges; }
-
- using const_iterator = RangeList::const_iterator;
- const_iterator begin() const { return Ranges.begin(); }
- const_iterator end() const { return Ranges.end(); }
-
- private:
- /// The instruction responsible for the access with respect to the local
- /// scope of the associated attribute.
- Instruction *LocalI;
-
- /// The instruction responsible for the access.
- Instruction *RemoteI;
-
- /// The value written, if any. `std::nullopt` means "not known yet",
- /// `nullptr` cannot be determined.
- std::optional<Value *> Content;
-
- /// Set of potential ranges accessed from the base pointer.
- RangeList Ranges;
-
- /// The access kind, e.g., READ, as bitset (could be more than one).
- AccessKind Kind;
-
- /// The type of the content, thus the type read/written, can be null if not
- /// available.
- Type *Ty;
- };
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAPointerInfo &createForPosition(const IRPosition &IRP, Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAPointerInfo"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- using OffsetBinsTy = DenseMap<AA::RangeTy, SmallSet<unsigned, 4>>;
- using const_bin_iterator = OffsetBinsTy::const_iterator;
- virtual const_bin_iterator begin() const = 0;
- virtual const_bin_iterator end() const = 0;
- virtual int64_t numOffsetBins() const = 0;
- virtual void dumpState(raw_ostream &O) const = 0;
- virtual const Access &getBinAccess(unsigned Index) const = 0;
-
- /// Call \p CB on all accesses that might interfere with \p Range and return
- /// true if all such accesses were known and the callback returned true for
- /// all of them, false otherwise. An access interferes with an offset-size
- /// pair if it might read or write that memory region.
- virtual bool forallInterferingAccesses(
- AA::RangeTy Range, function_ref<bool(const Access &, bool)> CB) const = 0;
-
- /// Call \p CB on all accesses that might interfere with \p I and
- /// return true if all such accesses were known and the callback returned true
- /// for all of them, false otherwise. In contrast to forallInterferingAccesses
- /// this function will perform reasoning to exclude write accesses that cannot
- /// affect the load even if they on the surface look as if they would. The
- /// flag \p HasBeenWrittenTo will be set to true if we know that \p I does not
- /// read the initial value of the underlying memory. If \p SkipCB is given and
- /// returns false for a potentially interfering access, that access is not
- /// checked for actual interference.
- virtual bool forallInterferingAccesses(
- Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
- bool FindInterferingWrites, bool FindInterferingReads,
- function_ref<bool(const Access &, bool)> CB, bool &HasBeenWrittenTo,
- AA::RangeTy &Range,
- function_ref<bool(const Access &)> SkipCB = nullptr) const = 0;
-
- /// This function should return true if the type of the \p AA is AAPointerInfo
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-raw_ostream &operator<<(raw_ostream &, const AAPointerInfo::Access &);
-
-/// An abstract attribute for getting assumption information.
-struct AAAssumptionInfo
- : public StateWrapper<SetState<StringRef>, AbstractAttribute,
- DenseSet<StringRef>> {
- using Base =
- StateWrapper<SetState<StringRef>, AbstractAttribute, DenseSet<StringRef>>;
-
- AAAssumptionInfo(const IRPosition &IRP, Attributor &A,
- const DenseSet<StringRef> &Known)
- : Base(IRP, Known) {}
-
- /// Returns true if the assumption set contains the assumption \p Assumption.
- virtual bool hasAssumption(const StringRef Assumption) const = 0;
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAAssumptionInfo &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAAssumptionInfo"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAAssumptionInfo
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract attribute for getting all assumption underlying objects.
-struct AAUnderlyingObjects : AbstractAttribute {
- AAUnderlyingObjects(const IRPosition &IRP) : AbstractAttribute(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return AbstractAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// See AbstractAttribute::requiresCallersForArgOrFunction
- static bool requiresCallersForArgOrFunction() { return true; }
-
- /// Create an abstract attribute biew for the position \p IRP.
- static AAUnderlyingObjects &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAUnderlyingObjects"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAUnderlyingObjects.
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-
- /// Check \p Pred on all underlying objects in \p Scope collected so far.
- ///
- /// This method will evaluate \p Pred on all underlying objects in \p Scope
- /// collected so far and return true if \p Pred holds on all of them.
- virtual bool
- forallUnderlyingObjects(function_ref<bool(Value &)> Pred,
- AA::ValueScope Scope = AA::Interprocedural) const = 0;
-};
-
-/// An abstract interface for address space information.
-struct AAAddressSpace : public StateWrapper<BooleanState, AbstractAttribute> {
- AAAddressSpace(const IRPosition &IRP, Attributor &A)
- : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return AbstractAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// See AbstractAttribute::requiresCallersForArgOrFunction
- static bool requiresCallersForArgOrFunction() { return true; }
-
- /// Return the address space of the associated value. \p NoAddressSpace is
- /// returned if the associated value is dead. This functions is not supposed
- /// to be called if the AA is invalid.
- virtual int32_t getAddressSpace() const = 0;
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAAddressSpace &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAAddressSpace"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAAssumptionInfo
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- // No address space which indicates the associated value is dead.
- static const int32_t NoAddressSpace = -1;
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-struct AAAllocationInfo : public StateWrapper<BooleanState, AbstractAttribute> {
- AAAllocationInfo(const IRPosition &IRP, Attributor &A)
- : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
- return false;
- return AbstractAttribute::isValidIRPositionForInit(A, IRP);
- }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAAllocationInfo &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- virtual std::optional<TypeSize> getAllocatedSize() const = 0;
-
- using NewOffsetsTy = DenseMap<AA::RangeTy, AA::RangeTy>;
- virtual const NewOffsetsTy &getNewOffsets() const = 0;
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAAllocationInfo"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAAllocationInfo
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- constexpr static const std::optional<TypeSize> HasNoAllocationSize =
- std::optional<TypeSize>(TypeSize(-1, true));
-
- static const char ID;
-};
-
-/// An abstract interface for llvm::GlobalValue information interference.
-struct AAGlobalValueInfo
- : public StateWrapper<BooleanState, AbstractAttribute> {
- AAGlobalValueInfo(const IRPosition &IRP, Attributor &A)
- : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (IRP.getPositionKind() != IRPosition::IRP_FLOAT)
- return false;
- auto *GV = dyn_cast<GlobalValue>(&IRP.getAnchorValue());
- if (!GV)
- return false;
- return GV->hasLocalLinkage();
- }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAGlobalValueInfo &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// Return true iff \p U is a potential use of the associated global value.
- virtual bool isPotentialUse(const Use &U) const = 0;
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAGlobalValueInfo"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAGlobalValueInfo
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract interface for indirect call information interference.
-struct AAIndirectCallInfo
- : public StateWrapper<BooleanState, AbstractAttribute> {
- AAIndirectCallInfo(const IRPosition &IRP, Attributor &A)
- : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
-
- /// See AbstractAttribute::isValidIRPositionForInit
- static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
- if (IRP.getPositionKind() != IRPosition::IRP_CALL_SITE)
- return false;
- auto *CB = cast<CallBase>(IRP.getCtxI());
- return CB->getOpcode() == Instruction::Call && CB->isIndirectCall() &&
- !CB->isMustTailCall();
- }
-
- /// Create an abstract attribute view for the position \p IRP.
- static AAIndirectCallInfo &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// Call \CB on each potential callee value and return true if all were known
- /// and \p CB returned true on all of them. Otherwise, return false.
- virtual bool foreachCallee(function_ref<bool(Function *)> CB) const = 0;
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AAIndirectCallInfo"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AAIndirectCallInfo
- /// This function should return true if the type of the \p AA is
- /// AADenormalFPMath.
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-/// An abstract Attribute for specializing "dynamic" components of
-/// "denormal-fp-math" and "denormal-fp-math-f32" to a known denormal mode.
-struct AADenormalFPMath
- : public StateWrapper<DenormalFPMathState, AbstractAttribute> {
- using Base = StateWrapper<DenormalFPMathState, AbstractAttribute>;
-
- AADenormalFPMath(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
-
- /// Create an abstract attribute view for the position \p IRP.
- static AADenormalFPMath &createForPosition(const IRPosition &IRP,
- Attributor &A);
-
- /// See AbstractAttribute::getName()
- const std::string getName() const override { return "AADenormalFPMath"; }
-
- /// See AbstractAttribute::getIdAddr()
- const char *getIdAddr() const override { return &ID; }
-
- /// This function should return true if the type of the \p AA is
- /// AADenormalFPMath.
- static bool classof(const AbstractAttribute *AA) {
- return (AA->getIdAddr() == &ID);
- }
-
- /// Unique ID (due to the unique address)
- static const char ID;
-};
-
-raw_ostream &operator<<(raw_ostream &, const AAPointerInfo::Access &);
-
-/// Run options, used by the pass manager.
-enum AttributorRunOption {
- NONE = 0,
- MODULE = 1 << 0,
- CGSCC = 1 << 1,
- ALL = MODULE | CGSCC
-};
-
-namespace AA {
-/// Helper to avoid creating an AA for IR Attributes that might already be set.
-template <Attribute::AttrKind AK, typename AAType = AbstractAttribute>
-bool hasAssumedIRAttr(Attributor &A, const AbstractAttribute *QueryingAA,
- const IRPosition &IRP, DepClassTy DepClass, bool &IsKnown,
- bool IgnoreSubsumingPositions = false,
- const AAType **AAPtr = nullptr) {
- IsKnown = false;
- switch (AK) {
-#define CASE(ATTRNAME, AANAME, ...) \
- case Attribute::ATTRNAME: { \
- if (AANAME::isImpliedByIR(A, IRP, AK, IgnoreSubsumingPositions)) \
- return IsKnown = true; \
- if (!QueryingAA) \
- return false; \
- const auto *AA = A.getAAFor<AANAME>(*QueryingAA, IRP, DepClass); \
- if (AAPtr) \
- *AAPtr = reinterpret_cast<const AAType *>(AA); \
- if (!AA || !AA->isAssumed(__VA_ARGS__)) \
- return false; \
- IsKnown = AA->isKnown(__VA_ARGS__); \
- return true; \
- }
- CASE(NoUnwind, AANoUnwind, );
- CASE(WillReturn, AAWillReturn, );
- CASE(NoFree, AANoFree, );
- CASE(NoCapture, AANoCapture, );
- CASE(NoRecurse, AANoRecurse, );
- CASE(NoReturn, AANoReturn, );
- CASE(NoSync, AANoSync, );
- CASE(NoAlias, AANoAlias, );
- CASE(NonNull, AANonNull, );
- CASE(MustProgress, AAMustProgress, );
- CASE(NoUndef, AANoUndef, );
- CASE(ReadNone, AAMemoryBehavior, AAMemoryBehavior::NO_ACCESSES);
- CASE(ReadOnly, AAMemoryBehavior, AAMemoryBehavior::NO_WRITES);
- CASE(WriteOnly, AAMemoryBehavior, AAMemoryBehavior::NO_READS);
-#undef CASE
- default:
- llvm_unreachable("hasAssumedIRAttr not available for this attribute kind");
- };
-}
-} // namespace AA
-
-} // end namespace llvm
-
-#endif // LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index dc985261ca7c6..e69de29bb2d1d 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -1,13216 +0,0 @@
-//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// See the Attributor.h file comment and the class descriptions in that file for
-// more information.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Transforms/IPO/Attributor.h"
-
-#include "llvm/ADT/APInt.h"
-#include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/DenseMapInfo.h"
-#include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/SCCIterator.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SetOperations.h"
-#include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Analysis/AssumeBundleQueries.h"
-#include "llvm/Analysis/AssumptionCache.h"
-#include "llvm/Analysis/CaptureTracking.h"
-#include "llvm/Analysis/CycleAnalysis.h"
-#include "llvm/Analysis/InstructionSimplify.h"
-#include "llvm/Analysis/LazyValueInfo.h"
-#include "llvm/Analysis/MemoryBuiltins.h"
-#include "llvm/Analysis/OptimizationRemarkEmitter.h"
-#include "llvm/Analysis/ScalarEvolution.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/Argument.h"
-#include "llvm/IR/Assumptions.h"
-#include "llvm/IR/Attributes.h"
-#include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/Constant.h"
-#include "llvm/IR/Constants.h"
-#include "llvm/IR/DataLayout.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/GlobalValue.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/InlineAsm.h"
-#include "llvm/IR/InstrTypes.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/IR/IntrinsicsAMDGPU.h"
-#include "llvm/IR/IntrinsicsNVPTX.h"
-#include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/MDBuilder.h"
-#include "llvm/IR/NoFolder.h"
-#include "llvm/IR/Value.h"
-#include "llvm/IR/ValueHandle.h"
-#include "llvm/Support/Alignment.h"
-#include "llvm/Support/Casting.h"
-#include "llvm/Support/CommandLine.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/GraphWriter.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Support/TypeSize.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-#include "llvm/Transforms/Utils/CallPromotionUtils.h"
-#include "llvm/Transforms/Utils/Local.h"
-#include "llvm/Transforms/Utils/ValueMapper.h"
-#include <cassert>
-#include <numeric>
-#include <optional>
-#include <string>
-
-using namespace llvm;
-
-#define DEBUG_TYPE "attributor"
-
-static cl::opt<bool> ManifestInternal(
- "attributor-manifest-internal", cl::Hidden,
- cl::desc("Manifest Attributor internal string attributes."),
- cl::init(false));
-
-static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(128),
- cl::Hidden);
-
-template <>
-unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;
-
-template <> unsigned llvm::PotentialLLVMValuesState::MaxPotentialValues = -1;
-
-static cl::opt<unsigned, true> MaxPotentialValues(
- "attributor-max-potential-values", cl::Hidden,
- cl::desc("Maximum number of potential values to be "
- "tracked for each position."),
- cl::location(llvm::PotentialConstantIntValuesState::MaxPotentialValues),
- cl::init(7));
-
-static cl::opt<int> MaxPotentialValuesIterations(
- "attributor-max-potential-values-iterations", cl::Hidden,
- cl::desc(
- "Maximum number of iterations we keep dismantling potential values."),
- cl::init(64));
-
-STATISTIC(NumAAs, "Number of abstract attributes created");
-
-// Some helper macros to deal with statistics tracking.
-//
-// Usage:
-// For simple IR attribute tracking overload trackStatistics in the abstract
-// attribute and choose the right STATS_DECLTRACK_********* macro,
-// e.g.,:
-// void trackStatistics() const override {
-// STATS_DECLTRACK_ARG_ATTR(returned)
-// }
-// If there is a single "increment" side one can use the macro
-// STATS_DECLTRACK with a custom message. If there are multiple increment
-// sides, STATS_DECL and STATS_TRACK can also be used separately.
-//
-#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
- ("Number of " #TYPE " marked '" #NAME "'")
-#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
-#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
-#define STATS_DECL(NAME, TYPE, MSG) \
- STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
-#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
-#define STATS_DECLTRACK(NAME, TYPE, MSG) \
- { \
- STATS_DECL(NAME, TYPE, MSG) \
- STATS_TRACK(NAME, TYPE) \
- }
-#define STATS_DECLTRACK_ARG_ATTR(NAME) \
- STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
-#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
- STATS_DECLTRACK(NAME, CSArguments, \
- BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
-#define STATS_DECLTRACK_FN_ATTR(NAME) \
- STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
-#define STATS_DECLTRACK_CS_ATTR(NAME) \
- STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
-#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
- STATS_DECLTRACK(NAME, FunctionReturn, \
- BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
-#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
- STATS_DECLTRACK(NAME, CSReturn, \
- BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
-#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
- STATS_DECLTRACK(NAME, Floating, \
- ("Number of floating values known to be '" #NAME "'"))
-
-// Specialization of the operator<< for abstract attributes subclasses. This
-// disambiguates situations where multiple operators are applicable.
-namespace llvm {
-#define PIPE_OPERATOR(CLASS) \
- raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
- return OS << static_cast<const AbstractAttribute &>(AA); \
- }
-
-PIPE_OPERATOR(AAIsDead)
-PIPE_OPERATOR(AANoUnwind)
-PIPE_OPERATOR(AANoSync)
-PIPE_OPERATOR(AANoRecurse)
-PIPE_OPERATOR(AANonConvergent)
-PIPE_OPERATOR(AAWillReturn)
-PIPE_OPERATOR(AANoReturn)
-PIPE_OPERATOR(AANonNull)
-PIPE_OPERATOR(AAMustProgress)
-PIPE_OPERATOR(AANoAlias)
-PIPE_OPERATOR(AADereferenceable)
-PIPE_OPERATOR(AAAlign)
-PIPE_OPERATOR(AAInstanceInfo)
-PIPE_OPERATOR(AANoCapture)
-PIPE_OPERATOR(AAValueSimplify)
-PIPE_OPERATOR(AANoFree)
-PIPE_OPERATOR(AAHeapToStack)
-PIPE_OPERATOR(AAIntraFnReachability)
-PIPE_OPERATOR(AAMemoryBehavior)
-PIPE_OPERATOR(AAMemoryLocation)
-PIPE_OPERATOR(AAValueConstantRange)
-PIPE_OPERATOR(AAPrivatizablePtr)
-PIPE_OPERATOR(AAUndefinedBehavior)
-PIPE_OPERATOR(AAPotentialConstantValues)
-PIPE_OPERATOR(AAPotentialValues)
-PIPE_OPERATOR(AANoUndef)
-PIPE_OPERATOR(AANoFPClass)
-PIPE_OPERATOR(AACallEdges)
-PIPE_OPERATOR(AAInterFnReachability)
-PIPE_OPERATOR(AAPointerInfo)
-PIPE_OPERATOR(AAAssumptionInfo)
-PIPE_OPERATOR(AAUnderlyingObjects)
-PIPE_OPERATOR(AAAddressSpace)
-PIPE_OPERATOR(AAAllocationInfo)
-PIPE_OPERATOR(AAIndirectCallInfo)
-PIPE_OPERATOR(AAGlobalValueInfo)
-PIPE_OPERATOR(AADenormalFPMath)
-
-#undef PIPE_OPERATOR
-
-template <>
-ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
- const DerefState &R) {
- ChangeStatus CS0 =
- clampStateAndIndicateChange(S.DerefBytesState, R.DerefBytesState);
- ChangeStatus CS1 = clampStateAndIndicateChange(S.GlobalState, R.GlobalState);
- return CS0 | CS1;
-}
-
-} // namespace llvm
-
-static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
- bool HeaderOnly, Cycle **CPtr = nullptr) {
- if (!CI)
- return true;
- auto *BB = I->getParent();
- auto *C = CI->getCycle(BB);
- if (!C)
- return false;
- if (CPtr)
- *CPtr = C;
- return !HeaderOnly || BB == C->getHeader();
-}
-
-/// Checks if a type could have padding bytes.
-static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
- // There is no size information, so be conservative.
- if (!Ty->isSized())
- return false;
-
- // If the alloc size is not equal to the storage size, then there are padding
- // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
- if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
- return false;
-
- // FIXME: This isn't the right way to check for padding in vectors with
- // non-byte-size elements.
- if (VectorType *SeqTy = dyn_cast<VectorType>(Ty))
- return isDenselyPacked(SeqTy->getElementType(), DL);
-
- // For array types, check for padding within members.
- if (ArrayType *SeqTy = dyn_cast<ArrayType>(Ty))
- return isDenselyPacked(SeqTy->getElementType(), DL);
-
- if (!isa<StructType>(Ty))
- return true;
-
- // Check for padding within and between elements of a struct.
- StructType *StructTy = cast<StructType>(Ty);
- const StructLayout *Layout = DL.getStructLayout(StructTy);
- uint64_t StartPos = 0;
- for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
- Type *ElTy = StructTy->getElementType(I);
- if (!isDenselyPacked(ElTy, DL))
- return false;
- if (StartPos != Layout->getElementOffsetInBits(I))
- return false;
- StartPos += DL.getTypeAllocSizeInBits(ElTy);
- }
-
- return true;
-}
-
-/// Get pointer operand of memory accessing instruction. If \p I is
-/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
-/// is set to false and the instruction is volatile, return nullptr.
-static const Value *getPointerOperand(const Instruction *I,
- bool AllowVolatile) {
- if (!AllowVolatile && I->isVolatile())
- return nullptr;
-
- if (auto *LI = dyn_cast<LoadInst>(I)) {
- return LI->getPointerOperand();
- }
-
- if (auto *SI = dyn_cast<StoreInst>(I)) {
- return SI->getPointerOperand();
- }
-
- if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(I)) {
- return CXI->getPointerOperand();
- }
-
- if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
- return RMWI->getPointerOperand();
- }
-
- return nullptr;
-}
-
-/// Helper function to create a pointer based on \p Ptr, and advanced by \p
-/// Offset bytes.
-static Value *constructPointer(Value *Ptr, int64_t Offset,
- IRBuilder<NoFolder> &IRB) {
- LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
- << "-bytes\n");
-
- if (Offset)
- Ptr = IRB.CreatePtrAdd(Ptr, IRB.getInt64(Offset),
- Ptr->getName() + ".b" + Twine(Offset));
- return Ptr;
-}
-
-static const Value *
-stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA,
- const Value *Val, const DataLayout &DL, APInt &Offset,
- bool GetMinOffset, bool AllowNonInbounds,
- bool UseAssumed = false) {
-
- auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
- const IRPosition &Pos = IRPosition::value(V);
- // Only track dependence if we are going to use the assumed info.
- const AAValueConstantRange *ValueConstantRangeAA =
- A.getAAFor<AAValueConstantRange>(QueryingAA, Pos,
- UseAssumed ? DepClassTy::OPTIONAL
- : DepClassTy::NONE);
- if (!ValueConstantRangeAA)
- return false;
- ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
- : ValueConstantRangeAA->getKnown();
- if (Range.isFullSet())
- return false;
-
- // We can only use the lower part of the range because the upper part can
- // be higher than what the value can really be.
- if (GetMinOffset)
- ROffset = Range.getSignedMin();
- else
- ROffset = Range.getSignedMax();
- return true;
- };
-
- return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
- /* AllowInvariant */ true,
- AttributorAnalysis);
-}
-
-static const Value *
-getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA,
- const Value *Ptr, int64_t &BytesOffset,
- const DataLayout &DL, bool AllowNonInbounds = false) {
- APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
- const Value *Base =
- stripAndAccumulateOffsets(A, QueryingAA, Ptr, DL, OffsetAPInt,
- /* GetMinOffset */ true, AllowNonInbounds);
-
- BytesOffset = OffsetAPInt.getSExtValue();
- return Base;
-}
-
-/// Clamp the information known for all returned values of a function
-/// (identified by \p QueryingAA) into \p S.
-template <typename AAType, typename StateType = typename AAType::StateType,
- Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
- bool RecurseForSelectAndPHI = true>
-static void clampReturnedValueStates(
- Attributor &A, const AAType &QueryingAA, StateType &S,
- const IRPosition::CallBaseContext *CBContext = nullptr) {
- LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
- << QueryingAA << " into " << S << "\n");
-
- assert((QueryingAA.getIRPosition().getPositionKind() ==
- IRPosition::IRP_RETURNED ||
- QueryingAA.getIRPosition().getPositionKind() ==
- IRPosition::IRP_CALL_SITE_RETURNED) &&
- "Can only clamp returned value states for a function returned or call "
- "site returned position!");
-
- // Use an optional state as there might not be any return values and we want
- // to join (IntegerState::operator&) the state of all there are.
- std::optional<StateType> T;
-
- // Callback for each possibly returned value.
- auto CheckReturnValue = [&](Value &RV) -> bool {
- const IRPosition &RVPos = IRPosition::value(RV, CBContext);
- // If possible, use the hasAssumedIRAttr interface.
- if (Attribute::isEnumAttrKind(IRAttributeKind)) {
- bool IsKnown;
- return AA::hasAssumedIRAttr<IRAttributeKind>(
- A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
- }
-
- const AAType *AA =
- A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
- if (!AA)
- return false;
- LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
- << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
- const StateType &AAS = AA->getState();
- if (!T)
- T = StateType::getBestState(AAS);
- *T &= AAS;
- LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
- << "\n");
- return T->isValidState();
- };
-
- if (!A.checkForAllReturnedValues(CheckReturnValue, QueryingAA,
- AA::ValueScope::Intraprocedural,
- RecurseForSelectAndPHI))
- S.indicatePessimisticFixpoint();
- else if (T)
- S ^= *T;
-}
-
-namespace {
-/// Helper class for generic deduction: return value -> returned position.
-template <typename AAType, typename BaseType,
- typename StateType = typename BaseType::StateType,
- bool PropagateCallBaseContext = false,
- Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
- bool RecurseForSelectAndPHI = true>
-struct AAReturnedFromReturnedValues : public BaseType {
- AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
- : BaseType(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- StateType S(StateType::getBestState(this->getState()));
- clampReturnedValueStates<AAType, StateType, IRAttributeKind, RecurseForSelectAndPHI>(
- A, *this, S,
- PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
- // TODO: If we know we visited all returned values, thus no are assumed
- // dead, we can take the known information from the state T.
- return clampStateAndIndicateChange<StateType>(this->getState(), S);
- }
-};
-
-/// Clamp the information known at all call sites for a given argument
-/// (identified by \p QueryingAA) into \p S.
-template <typename AAType, typename StateType = typename AAType::StateType,
- Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
-static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
- StateType &S) {
- LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
- << QueryingAA << " into " << S << "\n");
-
- assert(QueryingAA.getIRPosition().getPositionKind() ==
- IRPosition::IRP_ARGUMENT &&
- "Can only clamp call site argument states for an argument position!");
-
- // Use an optional state as there might not be any return values and we want
- // to join (IntegerState::operator&) the state of all there are.
- std::optional<StateType> T;
-
- // The argument number which is also the call site argument number.
- unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
-
- auto CallSiteCheck = [&](AbstractCallSite ACS) {
- const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
- // Check if a coresponding argument was found or if it is on not associated
- // (which can happen for callback calls).
- if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
- return false;
-
- // If possible, use the hasAssumedIRAttr interface.
- if (Attribute::isEnumAttrKind(IRAttributeKind)) {
- bool IsKnown;
- return AA::hasAssumedIRAttr<IRAttributeKind>(
- A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
- }
-
- const AAType *AA =
- A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
- if (!AA)
- return false;
- LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
- << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
- << "\n");
- const StateType &AAS = AA->getState();
- if (!T)
- T = StateType::getBestState(AAS);
- *T &= AAS;
- LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
- << "\n");
- return T->isValidState();
- };
-
- bool UsedAssumedInformation = false;
- if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
- UsedAssumedInformation))
- S.indicatePessimisticFixpoint();
- else if (T)
- S ^= *T;
-}
-
-/// This function is the bridge between argument position and the call base
-/// context.
-template <typename AAType, typename BaseType,
- typename StateType = typename AAType::StateType,
- Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
-bool getArgumentStateFromCallBaseContext(Attributor &A,
- BaseType &QueryingAttribute,
- IRPosition &Pos, StateType &State) {
- assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) &&
- "Expected an 'argument' position !");
- const CallBase *CBContext = Pos.getCallBaseContext();
- if (!CBContext)
- return false;
-
- int ArgNo = Pos.getCallSiteArgNo();
- assert(ArgNo >= 0 && "Invalid Arg No!");
- const IRPosition CBArgPos = IRPosition::callsite_argument(*CBContext, ArgNo);
-
- // If possible, use the hasAssumedIRAttr interface.
- if (Attribute::isEnumAttrKind(IRAttributeKind)) {
- bool IsKnown;
- return AA::hasAssumedIRAttr<IRAttributeKind>(
- A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
- }
-
- const auto *AA =
- A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
- if (!AA)
- return false;
- const StateType &CBArgumentState =
- static_cast<const StateType &>(AA->getState());
-
- LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
- << "Position:" << Pos << "CB Arg state:" << CBArgumentState
- << "\n");
-
- // NOTE: If we want to do call site grouping it should happen here.
- State ^= CBArgumentState;
- return true;
-}
-
-/// Helper class for generic deduction: call site argument -> argument position.
-template <typename AAType, typename BaseType,
- typename StateType = typename AAType::StateType,
- bool BridgeCallBaseContext = false,
- Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
-struct AAArgumentFromCallSiteArguments : public BaseType {
- AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
- : BaseType(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- StateType S = StateType::getBestState(this->getState());
-
- if (BridgeCallBaseContext) {
- bool Success =
- getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
- IRAttributeKind>(
- A, *this, this->getIRPosition(), S);
- if (Success)
- return clampStateAndIndicateChange<StateType>(this->getState(), S);
- }
- clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
- S);
-
- // TODO: If we know we visited all incoming values, thus no are assumed
- // dead, we can take the known information from the state T.
- return clampStateAndIndicateChange<StateType>(this->getState(), S);
- }
-};
-
-/// Helper class for generic replication: function returned -> cs returned.
-template <typename AAType, typename BaseType,
- typename StateType = typename BaseType::StateType,
- bool IntroduceCallBaseContext = false,
- Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
-struct AACalleeToCallSite : public BaseType {
- AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto IRPKind = this->getIRPosition().getPositionKind();
- assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED ||
- IRPKind == IRPosition::IRP_CALL_SITE) &&
- "Can only wrap function returned positions for call site "
- "returned positions!");
- auto &S = this->getState();
-
- CallBase &CB = cast<CallBase>(this->getAnchorValue());
- if (IntroduceCallBaseContext)
- LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
- << "\n");
-
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- auto CalleePred = [&](ArrayRef<const Function *> Callees) {
- for (const Function *Callee : Callees) {
- IRPosition FnPos =
- IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED
- ? IRPosition::returned(*Callee,
- IntroduceCallBaseContext ? &CB : nullptr)
- : IRPosition::function(
- *Callee, IntroduceCallBaseContext ? &CB : nullptr);
- // If possible, use the hasAssumedIRAttr interface.
- if (Attribute::isEnumAttrKind(IRAttributeKind)) {
- bool IsKnown;
- if (!AA::hasAssumedIRAttr<IRAttributeKind>(
- A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
- return false;
- continue;
- }
-
- const AAType *AA =
- A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
- if (!AA)
- return false;
- Changed |= clampStateAndIndicateChange(S, AA->getState());
- if (S.isAtFixpoint())
- return S.isValidState();
- }
- return true;
- };
- if (!A.checkForAllCallees(CalleePred, *this, CB))
- return S.indicatePessimisticFixpoint();
- return Changed;
- }
-};
-
-/// Helper function to accumulate uses.
-template <class AAType, typename StateType = typename AAType::StateType>
-static void followUsesInContext(AAType &AA, Attributor &A,
- MustBeExecutedContextExplorer &Explorer,
- const Instruction *CtxI,
- SetVector<const Use *> &Uses,
- StateType &State) {
- auto EIt = Explorer.begin(CtxI), EEnd = Explorer.end(CtxI);
- for (unsigned u = 0; u < Uses.size(); ++u) {
- const Use *U = Uses[u];
- if (const Instruction *UserI = dyn_cast<Instruction>(U->getUser())) {
- bool Found = Explorer.findInContextOf(UserI, EIt, EEnd);
- if (Found && AA.followUseInMBEC(A, U, UserI, State))
- for (const Use &Us : UserI->uses())
- Uses.insert(&Us);
- }
- }
-}
-
-/// Use the must-be-executed-context around \p I to add information into \p S.
-/// The AAType class is required to have `followUseInMBEC` method with the
-/// following signature and behaviour:
-///
-/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
-/// U - Underlying use.
-/// I - The user of the \p U.
-/// Returns true if the value should be tracked transitively.
-///
-template <class AAType, typename StateType = typename AAType::StateType>
-static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
- Instruction &CtxI) {
- MustBeExecutedContextExplorer *Explorer =
- A.getInfoCache().getMustBeExecutedContextExplorer();
- if (!Explorer)
- return;
-
- // Container for (transitive) uses of the associated value.
- SetVector<const Use *> Uses;
- for (const Use &U : AA.getIRPosition().getAssociatedValue().uses())
- Uses.insert(&U);
-
- followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
-
- if (S.isAtFixpoint())
- return;
-
- SmallVector<const BranchInst *, 4> BrInsts;
- auto Pred = [&](const Instruction *I) {
- if (const BranchInst *Br = dyn_cast<BranchInst>(I))
- if (Br->isConditional())
- BrInsts.push_back(Br);
- return true;
- };
-
- // Here, accumulate conditional branch instructions in the context. We
- // explore the child paths and collect the known states. The disjunction of
- // those states can be merged to its own state. Let ParentState_i be a state
- // to indicate the known information for an i-th branch instruction in the
- // context. ChildStates are created for its successors respectively.
- //
- // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
- // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
- // ...
- // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
- //
- // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
- //
- // FIXME: Currently, recursive branches are not handled. For example, we
- // can't deduce that ptr must be dereferenced in below function.
- //
- // void f(int a, int c, int *ptr) {
- // if(a)
- // if (b) {
- // *ptr = 0;
- // } else {
- // *ptr = 1;
- // }
- // else {
- // if (b) {
- // *ptr = 0;
- // } else {
- // *ptr = 1;
- // }
- // }
- // }
-
- Explorer->checkForAllContext(&CtxI, Pred);
- for (const BranchInst *Br : BrInsts) {
- StateType ParentState;
-
- // The known state of the parent state is a conjunction of children's
- // known states so it is initialized with a best state.
- ParentState.indicateOptimisticFixpoint();
-
- for (const BasicBlock *BB : Br->successors()) {
- StateType ChildState;
-
- size_t BeforeSize = Uses.size();
- followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
-
- // Erase uses which only appear in the child.
- for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
- It = Uses.erase(It);
-
- ParentState &= ChildState;
- }
-
- // Use only known state.
- S += ParentState;
- }
-}
-} // namespace
-
-/// ------------------------ PointerInfo ---------------------------------------
-
-namespace llvm {
-namespace AA {
-namespace PointerInfo {
-
-struct State;
-
-} // namespace PointerInfo
-} // namespace AA
-
-/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
-template <>
-struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> {
- using Access = AAPointerInfo::Access;
- static inline Access getEmptyKey();
- static inline Access getTombstoneKey();
- static unsigned getHashValue(const Access &A);
- static bool isEqual(const Access &LHS, const Access &RHS);
-};
-
-/// Helper that allows RangeTy as a key in a DenseMap.
-template <> struct DenseMapInfo<AA::RangeTy> {
- static inline AA::RangeTy getEmptyKey() {
- auto EmptyKey = DenseMapInfo<int64_t>::getEmptyKey();
- return AA::RangeTy{EmptyKey, EmptyKey};
- }
-
- static inline AA::RangeTy getTombstoneKey() {
- auto TombstoneKey = DenseMapInfo<int64_t>::getTombstoneKey();
- return AA::RangeTy{TombstoneKey, TombstoneKey};
- }
-
- static unsigned getHashValue(const AA::RangeTy &Range) {
- return detail::combineHashValue(
- DenseMapInfo<int64_t>::getHashValue(Range.Offset),
- DenseMapInfo<int64_t>::getHashValue(Range.Size));
- }
-
- static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
- return A == B;
- }
-};
-
-/// Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign
-/// but the instruction
-struct AccessAsInstructionInfo : DenseMapInfo<Instruction *> {
- using Base = DenseMapInfo<Instruction *>;
- using Access = AAPointerInfo::Access;
- static inline Access getEmptyKey();
- static inline Access getTombstoneKey();
- static unsigned getHashValue(const Access &A);
- static bool isEqual(const Access &LHS, const Access &RHS);
-};
-
-} // namespace llvm
-
-/// A type to track pointer/struct usage and accesses for AAPointerInfo.
-struct AA::PointerInfo::State : public AbstractState {
- /// Return the best possible representable state.
- static State getBestState(const State &SIS) { return State(); }
-
- /// Return the worst possible representable state.
- static State getWorstState(const State &SIS) {
- State R;
- R.indicatePessimisticFixpoint();
- return R;
- }
-
- State() = default;
- State(State &&SIS) = default;
-
- const State &getAssumed() const { return *this; }
-
- /// See AbstractState::isValidState().
- bool isValidState() const override { return BS.isValidState(); }
-
- /// See AbstractState::isAtFixpoint().
- bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
-
- /// See AbstractState::indicateOptimisticFixpoint().
- ChangeStatus indicateOptimisticFixpoint() override {
- BS.indicateOptimisticFixpoint();
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractState::indicatePessimisticFixpoint().
- ChangeStatus indicatePessimisticFixpoint() override {
- BS.indicatePessimisticFixpoint();
- return ChangeStatus::CHANGED;
- }
-
- State &operator=(const State &R) {
- if (this == &R)
- return *this;
- BS = R.BS;
- AccessList = R.AccessList;
- OffsetBins = R.OffsetBins;
- RemoteIMap = R.RemoteIMap;
- return *this;
- }
-
- State &operator=(State &&R) {
- if (this == &R)
- return *this;
- std::swap(BS, R.BS);
- std::swap(AccessList, R.AccessList);
- std::swap(OffsetBins, R.OffsetBins);
- std::swap(RemoteIMap, R.RemoteIMap);
- return *this;
- }
-
- /// Add a new Access to the state at offset \p Offset and with size \p Size.
- /// The access is associated with \p I, writes \p Content (if anything), and
- /// is of kind \p Kind. If an Access already exists for the same \p I and same
- /// \p RemoteI, the two are combined, potentially losing information about
- /// offset and size. The resulting access must now be moved from its original
- /// OffsetBin to the bin for its new offset.
- ///
- /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
- ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges,
- Instruction &I, std::optional<Value *> Content,
- AAPointerInfo::AccessKind Kind, Type *Ty,
- Instruction *RemoteI = nullptr);
-
- AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); }
- AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); }
- int64_t numOffsetBins() const { return OffsetBins.size(); }
-
- const AAPointerInfo::Access &getAccess(unsigned Index) const {
- return AccessList[Index];
- }
-
-protected:
- // Every memory instruction results in an Access object. We maintain a list of
- // all Access objects that we own, along with the following maps:
- //
- // - OffsetBins: RangeTy -> { Access }
- // - RemoteIMap: RemoteI x LocalI -> Access
- //
- // A RemoteI is any instruction that accesses memory. RemoteI is different
- // from LocalI if and only if LocalI is a call; then RemoteI is some
- // instruction in the callgraph starting from LocalI. Multiple paths in the
- // callgraph from LocalI to RemoteI may produce multiple accesses, but these
- // are all combined into a single Access object. This may result in loss of
- // information in RangeTy in the Access object.
- SmallVector<AAPointerInfo::Access> AccessList;
- AAPointerInfo::OffsetBinsTy OffsetBins;
- DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap;
-
- /// See AAPointerInfo::forallInterferingAccesses.
- bool forallInterferingAccesses(
- AA::RangeTy Range,
- function_ref<bool(const AAPointerInfo::Access &, bool)> CB) const {
- if (!isValidState())
- return false;
-
- for (const auto &It : OffsetBins) {
- AA::RangeTy ItRange = It.getFirst();
- if (!Range.mayOverlap(ItRange))
- continue;
- bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
- for (auto Index : It.getSecond()) {
- auto &Access = AccessList[Index];
- if (!CB(Access, IsExact))
- return false;
- }
- }
- return true;
- }
-
- /// See AAPointerInfo::forallInterferingAccesses.
- bool forallInterferingAccesses(
- Instruction &I,
- function_ref<bool(const AAPointerInfo::Access &, bool)> CB,
- AA::RangeTy &Range) const {
- if (!isValidState())
- return false;
-
- auto LocalList = RemoteIMap.find(&I);
- if (LocalList == RemoteIMap.end()) {
- return true;
- }
-
- for (unsigned Index : LocalList->getSecond()) {
- for (auto &R : AccessList[Index]) {
- Range &= R;
- if (Range.offsetAndSizeAreUnknown())
- break;
- }
- }
- return forallInterferingAccesses(Range, CB);
- }
-
-private:
- /// State to track fixpoint and validity.
- BooleanState BS;
-};
-
-ChangeStatus AA::PointerInfo::State::addAccess(
- Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I,
- std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
- Instruction *RemoteI) {
- RemoteI = RemoteI ? RemoteI : &I;
-
- // Check if we have an access for this instruction, if not, simply add it.
- auto &LocalList = RemoteIMap[RemoteI];
- bool AccExists = false;
- unsigned AccIndex = AccessList.size();
- for (auto Index : LocalList) {
- auto &A = AccessList[Index];
- if (A.getLocalInst() == &I) {
- AccExists = true;
- AccIndex = Index;
- break;
- }
- }
-
- auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
- LLVM_DEBUG(if (ToAdd.size()) dbgs()
- << "[AAPointerInfo] Inserting access in new offset bins\n";);
-
- for (auto Key : ToAdd) {
- LLVM_DEBUG(dbgs() << " key " << Key << "\n");
- OffsetBins[Key].insert(AccIndex);
- }
- };
-
- if (!AccExists) {
- AccessList.emplace_back(&I, RemoteI, Ranges, Content, Kind, Ty);
- assert((AccessList.size() == AccIndex + 1) &&
- "New Access should have been at AccIndex");
- LocalList.push_back(AccIndex);
- AddToBins(AccessList[AccIndex].getRanges());
- return ChangeStatus::CHANGED;
- }
-
- // Combine the new Access with the existing Access, and then update the
- // mapping in the offset bins.
- AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
- auto &Current = AccessList[AccIndex];
- auto Before = Current;
- Current &= Acc;
- if (Current == Before)
- return ChangeStatus::UNCHANGED;
-
- auto &ExistingRanges = Before.getRanges();
- auto &NewRanges = Current.getRanges();
-
- // Ranges that are in the old access but not the new access need to be removed
- // from the offset bins.
- AAPointerInfo::RangeList ToRemove;
- AAPointerInfo::RangeList::set_difference(ExistingRanges, NewRanges, ToRemove);
- LLVM_DEBUG(if (ToRemove.size()) dbgs()
- << "[AAPointerInfo] Removing access from old offset bins\n";);
-
- for (auto Key : ToRemove) {
- LLVM_DEBUG(dbgs() << " key " << Key << "\n");
- assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
- auto &Bin = OffsetBins[Key];
- assert(Bin.count(AccIndex) &&
- "Expected bin to actually contain the Access.");
- Bin.erase(AccIndex);
- }
-
- // Ranges that are in the new access but not the old access need to be added
- // to the offset bins.
- AAPointerInfo::RangeList ToAdd;
- AAPointerInfo::RangeList::set_difference(NewRanges, ExistingRanges, ToAdd);
- AddToBins(ToAdd);
- return ChangeStatus::CHANGED;
-}
-
-namespace {
-
-/// A helper containing a list of offsets computed for a Use. Ideally this
-/// list should be strictly ascending, but we ensure that only when we
-/// actually translate the list of offsets to a RangeList.
-struct OffsetInfo {
- using VecTy = SmallVector<int64_t>;
- using const_iterator = VecTy::const_iterator;
- VecTy Offsets;
-
- const_iterator begin() const { return Offsets.begin(); }
- const_iterator end() const { return Offsets.end(); }
-
- bool operator==(const OffsetInfo &RHS) const {
- return Offsets == RHS.Offsets;
- }
-
- bool operator!=(const OffsetInfo &RHS) const { return !(*this == RHS); }
-
- void insert(int64_t Offset) { Offsets.push_back(Offset); }
- bool isUnassigned() const { return Offsets.size() == 0; }
-
- bool isUnknown() const {
- if (isUnassigned())
- return false;
- if (Offsets.size() == 1)
- return Offsets.front() == AA::RangeTy::Unknown;
- return false;
- }
-
- void setUnknown() {
- Offsets.clear();
- Offsets.push_back(AA::RangeTy::Unknown);
- }
-
- void addToAll(int64_t Inc) {
- for (auto &Offset : Offsets) {
- Offset += Inc;
- }
- }
-
- /// Copy offsets from \p R into the current list.
- ///
- /// Ideally all lists should be strictly ascending, but we defer that to the
- /// actual use of the list. So we just blindly append here.
- void merge(const OffsetInfo &R) { Offsets.append(R.Offsets); }
-};
-
-#ifndef NDEBUG
-static raw_ostream &operator<<(raw_ostream &OS, const OffsetInfo &OI) {
- ListSeparator LS;
- OS << "[";
- for (auto Offset : OI) {
- OS << LS << Offset;
- }
- OS << "]";
- return OS;
-}
-#endif // NDEBUG
-
-struct AAPointerInfoImpl
- : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
- using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>;
- AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return std::string("PointerInfo ") +
- (isValidState() ? (std::string("#") +
- std::to_string(OffsetBins.size()) + " bins")
- : "<invalid>");
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- return AAPointerInfo::manifest(A);
- }
-
- virtual const_bin_iterator begin() const override { return State::begin(); }
- virtual const_bin_iterator end() const override { return State::end(); }
- virtual int64_t numOffsetBins() const override {
- return State::numOffsetBins();
- }
-
- virtual const Access &getBinAccess(unsigned Index) const override {
- return getAccess(Index);
- }
-
- bool forallInterferingAccesses(
- AA::RangeTy Range,
- function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
- const override {
- return State::forallInterferingAccesses(Range, CB);
- }
-
- bool forallInterferingAccesses(
- Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
- bool FindInterferingWrites, bool FindInterferingReads,
- function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
- AA::RangeTy &Range,
- function_ref<bool(const Access &)> SkipCB) const override {
- HasBeenWrittenTo = false;
-
- SmallPtrSet<const Access *, 8> DominatingWrites;
- SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
-
- Function &Scope = *I.getFunction();
- bool IsKnownNoSync;
- bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
- A, &QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL,
- IsKnownNoSync);
- const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
- IRPosition::function(Scope), &QueryingAA, DepClassTy::NONE);
- bool AllInSameNoSyncFn = IsAssumedNoSync;
- bool InstIsExecutedByInitialThreadOnly =
- ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
-
- // If the function is not ending in aligned barriers, we need the stores to
- // be in aligned barriers. The load being in one is not sufficient since the
- // store might be executed by a thread that disappears after, causing the
- // aligned barrier guarding the load to unblock and the load to read a value
- // that has no CFG path to the load.
- bool InstIsExecutedInAlignedRegion =
- FindInterferingReads && ExecDomainAA &&
- ExecDomainAA->isExecutedInAlignedRegion(A, I);
-
- if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
- A.recordDependence(*ExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
-
- InformationCache &InfoCache = A.getInfoCache();
- bool IsThreadLocalObj =
- AA::isAssumedThreadLocalObject(A, getAssociatedValue(), *this);
-
- // Helper to determine if we need to consider threading, which we cannot
- // right now. However, if the function is (assumed) nosync or the thread
- // executing all instructions is the main thread only we can ignore
- // threading. Also, thread-local objects do not require threading reasoning.
- // Finally, we can ignore threading if either access is executed in an
- // aligned region.
- auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
- if (IsThreadLocalObj || AllInSameNoSyncFn)
- return true;
- const auto *FnExecDomainAA =
- I.getFunction() == &Scope
- ? ExecDomainAA
- : A.lookupAAFor<AAExecutionDomain>(
- IRPosition::function(*I.getFunction()), &QueryingAA,
- DepClassTy::NONE);
- if (!FnExecDomainAA)
- return false;
- if (InstIsExecutedInAlignedRegion ||
- (FindInterferingWrites &&
- FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
- A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
- return true;
- }
- if (InstIsExecutedByInitialThreadOnly &&
- FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
- A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
- return true;
- }
- return false;
- };
-
- // Helper to determine if the access is executed by the same thread as the
- // given instruction, for now it is sufficient to avoid any potential
- // threading effects as we cannot deal with them anyway.
- auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
- return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
- (Acc.getRemoteInst() != Acc.getLocalInst() &&
- CanIgnoreThreadingForInst(*Acc.getLocalInst()));
- };
-
- // TODO: Use inter-procedural reachability and dominance.
- bool IsKnownNoRecurse;
- AA::hasAssumedIRAttr<Attribute::NoRecurse>(
- A, this, IRPosition::function(Scope), DepClassTy::OPTIONAL,
- IsKnownNoRecurse);
-
- // TODO: Use reaching kernels from AAKernelInfo (or move it to
- // AAExecutionDomain) such that we allow scopes other than kernels as long
- // as the reaching kernels are disjoint.
- bool InstInKernel = Scope.hasFnAttribute("kernel");
- bool ObjHasKernelLifetime = false;
- const bool UseDominanceReasoning =
- FindInterferingWrites && IsKnownNoRecurse;
- const DominatorTree *DT =
- InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(Scope);
-
- // Helper to check if a value has "kernel lifetime", that is it will not
- // outlive a GPU kernel. This is true for shared, constant, and local
- // globals on AMD and NVIDIA GPUs.
- auto HasKernelLifetime = [&](Value *V, Module &M) {
- if (!AA::isGPU(M))
- return false;
- switch (AA::GPUAddressSpace(V->getType()->getPointerAddressSpace())) {
- case AA::GPUAddressSpace::Shared:
- case AA::GPUAddressSpace::Constant:
- case AA::GPUAddressSpace::Local:
- return true;
- default:
- return false;
- };
- };
-
- // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
- // to determine if we should look at reachability from the callee. For
- // certain pointers we know the lifetime and we do not have to step into the
- // callee to determine reachability as the pointer would be dead in the
- // callee. See the conditional initialization below.
- std::function<bool(const Function &)> IsLiveInCalleeCB;
-
- if (auto *AI = dyn_cast<AllocaInst>(&getAssociatedValue())) {
- // If the alloca containing function is not recursive the alloca
- // must be dead in the callee.
- const Function *AIFn = AI->getFunction();
- ObjHasKernelLifetime = AIFn->hasFnAttribute("kernel");
- bool IsKnownNoRecurse;
- if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
- A, this, IRPosition::function(*AIFn), DepClassTy::OPTIONAL,
- IsKnownNoRecurse)) {
- IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
- }
- } else if (auto *GV = dyn_cast<GlobalValue>(&getAssociatedValue())) {
- // If the global has kernel lifetime we can stop if we reach a kernel
- // as it is "dead" in the (unknown) callees.
- ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
- if (ObjHasKernelLifetime)
- IsLiveInCalleeCB = [](const Function &Fn) {
- return !Fn.hasFnAttribute("kernel");
- };
- }
-
- // Set of accesses/instructions that will overwrite the result and are
- // therefore blockers in the reachability traversal.
- AA::InstExclusionSetTy ExclusionSet;
-
- auto AccessCB = [&](const Access &Acc, bool Exact) {
- Function *AccScope = Acc.getRemoteInst()->getFunction();
- bool AccInSameScope = AccScope == &Scope;
-
- // If the object has kernel lifetime we can ignore accesses only reachable
- // by other kernels. For now we only skip accesses *in* other kernels.
- if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
- AccScope->hasFnAttribute("kernel"))
- return true;
-
- if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
- if (Acc.isWrite() || (isa<LoadInst>(I) && Acc.isWriteOrAssumption()))
- ExclusionSet.insert(Acc.getRemoteInst());
- }
-
- if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
- (!FindInterferingReads || !Acc.isRead()))
- return true;
-
- bool Dominates = FindInterferingWrites && DT && Exact &&
- Acc.isMustAccess() && AccInSameScope &&
- DT->dominates(Acc.getRemoteInst(), &I);
- if (Dominates)
- DominatingWrites.insert(&Acc);
-
- // Track if all interesting accesses are in the same `nosync` function as
- // the given instruction.
- AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
-
- InterferingAccesses.push_back({&Acc, Exact});
- return true;
- };
- if (!State::forallInterferingAccesses(I, AccessCB, Range))
- return false;
-
- HasBeenWrittenTo = !DominatingWrites.empty();
-
- // Dominating writes form a chain, find the least/lowest member.
- Instruction *LeastDominatingWriteInst = nullptr;
- for (const Access *Acc : DominatingWrites) {
- if (!LeastDominatingWriteInst) {
- LeastDominatingWriteInst = Acc->getRemoteInst();
- } else if (DT->dominates(LeastDominatingWriteInst,
- Acc->getRemoteInst())) {
- LeastDominatingWriteInst = Acc->getRemoteInst();
- }
- }
-
- // Helper to determine if we can skip a specific write access.
- auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
- if (SkipCB && SkipCB(Acc))
- return true;
- if (!CanIgnoreThreading(Acc))
- return false;
-
- // Check read (RAW) dependences and write (WAR) dependences as necessary.
- // If we successfully excluded all effects we are interested in, the
- // access can be skipped.
- bool ReadChecked = !FindInterferingReads;
- bool WriteChecked = !FindInterferingWrites;
-
- // If the instruction cannot reach the access, the former does not
- // interfere with what the access reads.
- if (!ReadChecked) {
- if (!AA::isPotentiallyReachable(A, I, *Acc.getRemoteInst(), QueryingAA,
- &ExclusionSet, IsLiveInCalleeCB))
- ReadChecked = true;
- }
- // If the instruction cannot be reach from the access, the latter does not
- // interfere with what the instruction reads.
- if (!WriteChecked) {
- if (!AA::isPotentiallyReachable(A, *Acc.getRemoteInst(), I, QueryingAA,
- &ExclusionSet, IsLiveInCalleeCB))
- WriteChecked = true;
- }
-
- // If we still might be affected by the write of the access but there are
- // dominating writes in the function of the instruction
- // (HasBeenWrittenTo), we can try to reason that the access is overwritten
- // by them. This would have happend above if they are all in the same
- // function, so we only check the inter-procedural case. Effectively, we
- // want to show that there is no call after the dominting write that might
- // reach the access, and when it returns reach the instruction with the
- // updated value. To this end, we iterate all call sites, check if they
- // might reach the instruction without going through another access
- // (ExclusionSet) and at the same time might reach the access. However,
- // that is all part of AAInterFnReachability.
- if (!WriteChecked && HasBeenWrittenTo &&
- Acc.getRemoteInst()->getFunction() != &Scope) {
-
- const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
- QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL);
-
- // Without going backwards in the call tree, can we reach the access
- // from the least dominating write. Do not allow to pass the instruction
- // itself either.
- bool Inserted = ExclusionSet.insert(&I).second;
-
- if (!FnReachabilityAA ||
- !FnReachabilityAA->instructionCanReach(
- A, *LeastDominatingWriteInst,
- *Acc.getRemoteInst()->getFunction(), &ExclusionSet))
- WriteChecked = true;
-
- if (Inserted)
- ExclusionSet.erase(&I);
- }
-
- if (ReadChecked && WriteChecked)
- return true;
-
- if (!DT || !UseDominanceReasoning)
- return false;
- if (!DominatingWrites.count(&Acc))
- return false;
- return LeastDominatingWriteInst != Acc.getRemoteInst();
- };
-
- // Run the user callback on all accesses we cannot skip and return if
- // that succeeded for all or not.
- for (auto &It : InterferingAccesses) {
- if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
- !CanSkipAccess(*It.first, It.second)) {
- if (!UserCB(*It.first, It.second))
- return false;
- }
- }
- return true;
- }
-
- ChangeStatus translateAndAddStateFromCallee(Attributor &A,
- const AAPointerInfo &OtherAA,
- CallBase &CB) {
- using namespace AA::PointerInfo;
- if (!OtherAA.getState().isValidState() || !isValidState())
- return indicatePessimisticFixpoint();
-
- const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
- bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
-
- // Combine the accesses bin by bin.
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- const auto &State = OtherAAImpl.getState();
- for (const auto &It : State) {
- for (auto Index : It.getSecond()) {
- const auto &RAcc = State.getAccess(Index);
- if (IsByval && !RAcc.isRead())
- continue;
- bool UsedAssumedInformation = false;
- AccessKind AK = RAcc.getKind();
- auto Content = A.translateArgumentToCallSiteContent(
- RAcc.getContent(), CB, *this, UsedAssumedInformation);
- AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
- AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
-
- Changed |= addAccess(A, RAcc.getRanges(), CB, Content, AK,
- RAcc.getType(), RAcc.getRemoteInst());
- }
- }
- return Changed;
- }
-
- ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
- const OffsetInfo &Offsets, CallBase &CB) {
- using namespace AA::PointerInfo;
- if (!OtherAA.getState().isValidState() || !isValidState())
- return indicatePessimisticFixpoint();
-
- const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
-
- // Combine the accesses bin by bin.
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- const auto &State = OtherAAImpl.getState();
- for (const auto &It : State) {
- for (auto Index : It.getSecond()) {
- const auto &RAcc = State.getAccess(Index);
- for (auto Offset : Offsets) {
- auto NewRanges = Offset == AA::RangeTy::Unknown
- ? AA::RangeTy::getUnknown()
- : RAcc.getRanges();
- if (!NewRanges.isUnknown()) {
- NewRanges.addToAllOffsets(Offset);
- }
- Changed |=
- addAccess(A, NewRanges, CB, RAcc.getContent(), RAcc.getKind(),
- RAcc.getType(), RAcc.getRemoteInst());
- }
- }
- }
- return Changed;
- }
-
- /// Statistic tracking for all AAPointerInfo implementations.
- /// See AbstractAttribute::trackStatistics().
- void trackPointerInfoStatistics(const IRPosition &IRP) const {}
-
- /// Dump the state into \p O.
- virtual void dumpState(raw_ostream &O) const override {
- for (auto &It : OffsetBins) {
- O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
- << "] : " << It.getSecond().size() << "\n";
- for (auto AccIndex : It.getSecond()) {
- auto &Acc = AccessList[AccIndex];
- O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
- if (Acc.getLocalInst() != Acc.getRemoteInst())
- O << " --> " << *Acc.getRemoteInst()
- << "\n";
- if (!Acc.isWrittenValueYetUndetermined()) {
- if (isa_and_nonnull<Function>(Acc.getWrittenValue()))
- O << " - c: func " << Acc.getWrittenValue()->getName()
- << "\n";
- else if (Acc.getWrittenValue())
- O << " - c: " << *Acc.getWrittenValue() << "\n";
- else
- O << " - c: <unknown>\n";
- }
- }
- }
- }
-};
-
-struct AAPointerInfoFloating : public AAPointerInfoImpl {
- using AccessKind = AAPointerInfo::AccessKind;
- AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
- : AAPointerInfoImpl(IRP, A) {}
-
- /// Deal with an access and signal if it was handled successfully.
- bool handleAccess(Attributor &A, Instruction &I,
- std::optional<Value *> Content, AccessKind Kind,
- SmallVectorImpl<int64_t> &Offsets, ChangeStatus &Changed,
- Type &Ty) {
- using namespace AA::PointerInfo;
- auto Size = AA::RangeTy::Unknown;
- const DataLayout &DL = A.getDataLayout();
- TypeSize AccessSize = DL.getTypeStoreSize(&Ty);
- if (!AccessSize.isScalable())
- Size = AccessSize.getFixedValue();
-
- // Make a strictly ascending list of offsets as required by addAccess()
- llvm::sort(Offsets);
- auto *Last = std::unique(Offsets.begin(), Offsets.end());
- Offsets.erase(Last, Offsets.end());
-
- VectorType *VT = dyn_cast<VectorType>(&Ty);
- if (!VT || VT->getElementCount().isScalable() ||
- !Content.value_or(nullptr) || !isa<Constant>(*Content) ||
- (*Content)->getType() != VT ||
- DL.getTypeStoreSize(VT->getElementType()).isScalable()) {
- Changed = Changed | addAccess(A, {Offsets, Size}, I, Content, Kind, &Ty);
- } else {
- // Handle vector stores with constant content element-wise.
- // TODO: We could look for the elements or create instructions
- // representing them.
- // TODO: We need to push the Content into the range abstraction
- // (AA::RangeTy) to allow different content values for different
- // ranges. ranges. Hence, support vectors storing different values.
- Type *ElementType = VT->getElementType();
- int64_t ElementSize = DL.getTypeStoreSize(ElementType).getFixedValue();
- auto *ConstContent = cast<Constant>(*Content);
- Type *Int32Ty = Type::getInt32Ty(ElementType->getContext());
- SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());
-
- for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
- Value *ElementContent = ConstantExpr::getExtractElement(
- ConstContent, ConstantInt::get(Int32Ty, i));
-
- // Add the element access.
- Changed = Changed | addAccess(A, {ElementOffsets, ElementSize}, I,
- ElementContent, Kind, ElementType);
-
- // Advance the offsets for the next element.
- for (auto &ElementOffset : ElementOffsets)
- ElementOffset += ElementSize;
- }
- }
- return true;
- };
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override;
-
- /// If the indices to \p GEP can be traced to constants, incorporate all
- /// of these into \p UsrOI.
- ///
- /// \return true iff \p UsrOI is updated.
- bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
- OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
- const GEPOperator *GEP);
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
- }
-};
-
-bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
- const DataLayout &DL,
- OffsetInfo &UsrOI,
- const OffsetInfo &PtrOI,
- const GEPOperator *GEP) {
- unsigned BitWidth = DL.getIndexTypeSizeInBits(GEP->getType());
- MapVector<Value *, APInt> VariableOffsets;
- APInt ConstantOffset(BitWidth, 0);
-
- assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
- "Don't look for constant values if the offset has already been "
- "determined to be unknown.");
-
- if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
- UsrOI.setUnknown();
- return true;
- }
-
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
- << (VariableOffsets.empty() ? "" : "not") << " constant "
- << *GEP << "\n");
-
- auto Union = PtrOI;
- Union.addToAll(ConstantOffset.getSExtValue());
-
- // Each VI in VariableOffsets has a set of potential constant values. Every
- // combination of elements, picked one each from these sets, is separately
- // added to the original set of offsets, thus resulting in more offsets.
- for (const auto &VI : VariableOffsets) {
- auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
- *this, IRPosition::value(*VI.first), DepClassTy::OPTIONAL);
- if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
- UsrOI.setUnknown();
- return true;
- }
-
- // UndefValue is treated as a zero, which leaves Union as is.
- if (PotentialConstantsAA->undefIsContained())
- continue;
-
- // We need at least one constant in every set to compute an actual offset.
- // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
- // don't actually exist. In other words, the absence of constant values
- // implies that the operation can be assumed dead for now.
- auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
- if (AssumedSet.empty())
- return false;
-
- OffsetInfo Product;
- for (const auto &ConstOffset : AssumedSet) {
- auto CopyPerOffset = Union;
- CopyPerOffset.addToAll(ConstOffset.getSExtValue() *
- VI.second.getZExtValue());
- Product.merge(CopyPerOffset);
- }
- Union = Product;
- }
-
- UsrOI = std::move(Union);
- return true;
-}
-
-ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
- using namespace AA::PointerInfo;
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- const DataLayout &DL = A.getDataLayout();
- Value &AssociatedValue = getAssociatedValue();
-
- DenseMap<Value *, OffsetInfo> OffsetInfoMap;
- OffsetInfoMap[&AssociatedValue].insert(0);
-
- auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
- // One does not simply walk into a map and assign a reference to a possibly
- // new location. That can cause an invalidation before the assignment
- // happens, like so:
- //
- // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
- //
- // The RHS is a reference that may be invalidated by an insertion caused by
- // the LHS. So we ensure that the side-effect of the LHS happens first.
- auto &UsrOI = OffsetInfoMap[Usr];
- auto &PtrOI = OffsetInfoMap[CurPtr];
- assert(!PtrOI.isUnassigned() &&
- "Cannot pass through if the input Ptr was not visited!");
- UsrOI = PtrOI;
- Follow = true;
- return true;
- };
-
- const auto *F = getAnchorScope();
- const auto *CI =
- F ? A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(*F)
- : nullptr;
- const auto *TLI =
- F ? A.getInfoCache().getTargetLibraryInfoForFunction(*F) : nullptr;
-
- auto UsePred = [&](const Use &U, bool &Follow) -> bool {
- Value *CurPtr = U.get();
- User *Usr = U.getUser();
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
- << "\n");
- assert(OffsetInfoMap.count(CurPtr) &&
- "The current pointer offset should have been seeded!");
-
- if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Usr)) {
- if (CE->isCast())
- return HandlePassthroughUser(Usr, CurPtr, Follow);
- if (CE->isCompare())
- return true;
- if (!isa<GEPOperator>(CE)) {
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE
- << "\n");
- return false;
- }
- }
- if (auto *GEP = dyn_cast<GEPOperator>(Usr)) {
- // Note the order here, the Usr access might change the map, CurPtr is
- // already in it though.
- auto &UsrOI = OffsetInfoMap[Usr];
- auto &PtrOI = OffsetInfoMap[CurPtr];
-
- if (UsrOI.isUnknown())
- return true;
-
- if (PtrOI.isUnknown()) {
- Follow = true;
- UsrOI.setUnknown();
- return true;
- }
-
- Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
- return true;
- }
- if (isa<PtrToIntInst>(Usr))
- return false;
- if (isa<CastInst>(Usr) || isa<SelectInst>(Usr) || isa<ReturnInst>(Usr))
- return HandlePassthroughUser(Usr, CurPtr, Follow);
-
- // For PHIs we need to take care of the recurrence explicitly as the value
- // might change while we iterate through a loop. For now, we give up if
- // the PHI is not invariant.
- if (isa<PHINode>(Usr)) {
- // Note the order here, the Usr access might change the map, CurPtr is
- // already in it though.
- bool IsFirstPHIUser = !OffsetInfoMap.count(Usr);
- auto &UsrOI = OffsetInfoMap[Usr];
- auto &PtrOI = OffsetInfoMap[CurPtr];
-
- // Check if the PHI operand has already an unknown offset as we can't
- // improve on that anymore.
- if (PtrOI.isUnknown()) {
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
- << *CurPtr << " in " << *Usr << "\n");
- Follow = !UsrOI.isUnknown();
- UsrOI.setUnknown();
- return true;
- }
-
- // Check if the PHI is invariant (so far).
- if (UsrOI == PtrOI) {
- assert(!PtrOI.isUnassigned() &&
- "Cannot assign if the current Ptr was not visited!");
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
- return true;
- }
-
- // Check if the PHI operand can be traced back to AssociatedValue.
- APInt Offset(
- DL.getIndexSizeInBits(CurPtr->getType()->getPointerAddressSpace()),
- 0);
- Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
- DL, Offset, /* AllowNonInbounds */ true);
- auto It = OffsetInfoMap.find(CurPtrBase);
- if (It == OffsetInfoMap.end()) {
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
- << *CurPtr << " in " << *Usr << "\n");
- UsrOI.setUnknown();
- Follow = true;
- return true;
- }
-
- // Check if the PHI operand is not dependent on the PHI itself. Every
- // recurrence is a cyclic net of PHIs in the data flow, and has an
- // equivalent Cycle in the control flow. One of those PHIs must be in the
- // header of that control flow Cycle. This is independent of the choice of
- // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
- // every Cycle header; if such a node is marked unknown, this will
- // eventually propagate through the whole net of PHIs in the recurrence.
- if (mayBeInCycle(CI, cast<Instruction>(Usr), /* HeaderOnly */ true)) {
- auto BaseOI = It->getSecond();
- BaseOI.addToAll(Offset.getZExtValue());
- if (IsFirstPHIUser || BaseOI == UsrOI) {
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
- << " in " << *Usr << "\n");
- return HandlePassthroughUser(Usr, CurPtr, Follow);
- }
-
- LLVM_DEBUG(
- dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
- << *CurPtr << " in " << *Usr << "\n");
- UsrOI.setUnknown();
- Follow = true;
- return true;
- }
-
- UsrOI.merge(PtrOI);
- Follow = true;
- return true;
- }
-
- if (auto *LoadI = dyn_cast<LoadInst>(Usr)) {
- // If the access is to a pointer that may or may not be the associated
- // value, e.g. due to a PHI, we cannot assume it will be read.
- AccessKind AK = AccessKind::AK_R;
- if (getUnderlyingObject(CurPtr) == &AssociatedValue)
- AK = AccessKind(AK | AccessKind::AK_MUST);
- else
- AK = AccessKind(AK | AccessKind::AK_MAY);
- if (!handleAccess(A, *LoadI, /* Content */ nullptr, AK,
- OffsetInfoMap[CurPtr].Offsets, Changed,
- *LoadI->getType()))
- return false;
-
- auto IsAssumption = [](Instruction &I) {
- if (auto *II = dyn_cast<IntrinsicInst>(&I))
- return II->isAssumeLikeIntrinsic();
- return false;
- };
-
- auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
- // Check if the assumption and the load are executed together without
- // memory modification.
- do {
- if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
- return true;
- FromI = FromI->getNextNonDebugInstruction();
- } while (FromI && FromI != ToI);
- return false;
- };
-
- BasicBlock *BB = LoadI->getParent();
- auto IsValidAssume = [&](IntrinsicInst &IntrI) {
- if (IntrI.getIntrinsicID() != Intrinsic::assume)
- return false;
- BasicBlock *IntrBB = IntrI.getParent();
- if (IntrI.getParent() == BB) {
- if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(), &IntrI))
- return false;
- } else {
- auto PredIt = pred_begin(IntrBB);
- if (PredIt == pred_end(IntrBB))
- return false;
- if ((*PredIt) != BB)
- return false;
- if (++PredIt != pred_end(IntrBB))
- return false;
- for (auto *SuccBB : successors(BB)) {
- if (SuccBB == IntrBB)
- continue;
- if (isa<UnreachableInst>(SuccBB->getTerminator()))
- continue;
- return false;
- }
- if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(),
- BB->getTerminator()))
- return false;
- if (IsImpactedInRange(&IntrBB->front(), &IntrI))
- return false;
- }
- return true;
- };
-
- std::pair<Value *, IntrinsicInst *> Assumption;
- for (const Use &LoadU : LoadI->uses()) {
- if (auto *CmpI = dyn_cast<CmpInst>(LoadU.getUser())) {
- if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
- continue;
- for (const Use &CmpU : CmpI->uses()) {
- if (auto *IntrI = dyn_cast<IntrinsicInst>(CmpU.getUser())) {
- if (!IsValidAssume(*IntrI))
- continue;
- int Idx = CmpI->getOperandUse(0) == LoadU;
- Assumption = {CmpI->getOperand(Idx), IntrI};
- break;
- }
- }
- }
- if (Assumption.first)
- break;
- }
-
- // Check if we found an assumption associated with this load.
- if (!Assumption.first || !Assumption.second)
- return true;
-
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
- << *Assumption.second << ": " << *LoadI
- << " == " << *Assumption.first << "\n");
- bool UsedAssumedInformation = false;
- std::optional<Value *> Content = nullptr;
- if (Assumption.first)
- Content =
- A.getAssumedSimplified(*Assumption.first, *this,
- UsedAssumedInformation, AA::Interprocedural);
- return handleAccess(
- A, *Assumption.second, Content, AccessKind::AK_ASSUMPTION,
- OffsetInfoMap[CurPtr].Offsets, Changed, *LoadI->getType());
- }
-
- auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
- ArrayRef<Value *> OtherOps, AccessKind AK) {
- for (auto *OtherOp : OtherOps) {
- if (OtherOp == CurPtr) {
- LLVM_DEBUG(
- dbgs()
- << "[AAPointerInfo] Escaping use in store like instruction " << I
- << "\n");
- return false;
- }
- }
-
- // If the access is to a pointer that may or may not be the associated
- // value, e.g. due to a PHI, we cannot assume it will be written.
- if (getUnderlyingObject(CurPtr) == &AssociatedValue)
- AK = AccessKind(AK | AccessKind::AK_MUST);
- else
- AK = AccessKind(AK | AccessKind::AK_MAY);
- bool UsedAssumedInformation = false;
- std::optional<Value *> Content = nullptr;
- if (ValueOp)
- Content = A.getAssumedSimplified(
- *ValueOp, *this, UsedAssumedInformation, AA::Interprocedural);
- return handleAccess(A, I, Content, AK, OffsetInfoMap[CurPtr].Offsets,
- Changed, ValueTy);
- };
-
- if (auto *StoreI = dyn_cast<StoreInst>(Usr))
- return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
- *StoreI->getValueOperand()->getType(),
- {StoreI->getValueOperand()}, AccessKind::AK_W);
- if (auto *RMWI = dyn_cast<AtomicRMWInst>(Usr))
- return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
- {RMWI->getValOperand()}, AccessKind::AK_RW);
- if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Usr))
- return HandleStoreLike(
- *CXI, nullptr, *CXI->getNewValOperand()->getType(),
- {CXI->getCompareOperand(), CXI->getNewValOperand()},
- AccessKind::AK_RW);
-
- if (auto *CB = dyn_cast<CallBase>(Usr)) {
- if (CB->isLifetimeStartOrEnd())
- return true;
- if (getFreedOperand(CB, TLI) == U)
- return true;
- if (CB->isArgOperand(&U)) {
- unsigned ArgNo = CB->getArgOperandNo(&U);
- const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
- *this, IRPosition::callsite_argument(*CB, ArgNo),
- DepClassTy::REQUIRED);
- if (!CSArgPI)
- return false;
- Changed =
- translateAndAddState(A, *CSArgPI, OffsetInfoMap[CurPtr], *CB) |
- Changed;
- return isValidState();
- }
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
- << "\n");
- // TODO: Allow some call uses
- return false;
- }
-
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
- return false;
- };
- auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
- assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
- if (OffsetInfoMap.count(NewU)) {
- LLVM_DEBUG({
- if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
- dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
- << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
- << "\n";
- }
- });
- return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
- }
- OffsetInfoMap[NewU] = OffsetInfoMap[OldU];
- return true;
- };
- if (!A.checkForAllUses(UsePred, *this, AssociatedValue,
- /* CheckBBLivenessOnly */ true, DepClassTy::OPTIONAL,
- /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
- return indicatePessimisticFixpoint();
- }
-
- LLVM_DEBUG({
- dbgs() << "Accesses by bin after update:\n";
- dumpState(dbgs());
- });
-
- return Changed;
-}
-
-struct AAPointerInfoReturned final : AAPointerInfoImpl {
- AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
- : AAPointerInfoImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- return indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
- }
-};
-
-struct AAPointerInfoArgument final : AAPointerInfoFloating {
- AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
- : AAPointerInfoFloating(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
- }
-};
-
-struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
- AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAPointerInfoFloating(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- using namespace AA::PointerInfo;
- // We handle memory intrinsics explicitly, at least the first (=
- // destination) and second (=source) arguments as we know how they are
- // accessed.
- if (auto *MI = dyn_cast_or_null<MemIntrinsic>(getCtxI())) {
- ConstantInt *Length = dyn_cast<ConstantInt>(MI->getLength());
- int64_t LengthVal = AA::RangeTy::Unknown;
- if (Length)
- LengthVal = Length->getSExtValue();
- unsigned ArgNo = getIRPosition().getCallSiteArgNo();
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- if (ArgNo > 1) {
- LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
- << *MI << "\n");
- return indicatePessimisticFixpoint();
- } else {
- auto Kind =
- ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
- Changed =
- Changed | addAccess(A, {0, LengthVal}, *MI, nullptr, Kind, nullptr);
- }
- LLVM_DEBUG({
- dbgs() << "Accesses by bin after update:\n";
- dumpState(dbgs());
- });
-
- return Changed;
- }
-
- // TODO: Once we have call site specific value information we can provide
- // call site specific liveness information and then it makes
- // sense to specialize attributes for call sites arguments instead of
- // redirecting requests to the callee argument.
- Argument *Arg = getAssociatedArgument();
- if (Arg) {
- const IRPosition &ArgPos = IRPosition::argument(*Arg);
- auto *ArgAA =
- A.getAAFor<AAPointerInfo>(*this, ArgPos, DepClassTy::REQUIRED);
- if (ArgAA && ArgAA->getState().isValidState())
- return translateAndAddStateFromCallee(A, *ArgAA,
- *cast<CallBase>(getCtxI()));
- if (!Arg->getParent()->isDeclaration())
- return indicatePessimisticFixpoint();
- }
-
- bool IsKnownNoCapture;
- if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoCapture))
- return indicatePessimisticFixpoint();
-
- bool IsKnown = false;
- if (AA::isAssumedReadNone(A, getIRPosition(), *this, IsKnown))
- return ChangeStatus::UNCHANGED;
- bool ReadOnly = AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown);
- auto Kind =
- ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
- return addAccess(A, AA::RangeTy::getUnknown(), *getCtxI(), nullptr, Kind,
- nullptr);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
- }
-};
-
-struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
- AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AAPointerInfoFloating(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
- }
-};
-} // namespace
-
-/// -----------------------NoUnwind Function Attribute--------------------------
-
-namespace {
-struct AANoUnwindImpl : AANoUnwind {
- AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- bool IsKnown;
- assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
- A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
- (void)IsKnown;
- }
-
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "nounwind" : "may-unwind";
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto Opcodes = {
- (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
- (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
- (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
-
- auto CheckForNoUnwind = [&](Instruction &I) {
- if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
- return true;
-
- if (const auto *CB = dyn_cast<CallBase>(&I)) {
- bool IsKnownNoUnwind;
- return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
- A, this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED,
- IsKnownNoUnwind);
- }
- return false;
- };
-
- bool UsedAssumedInformation = false;
- if (!A.checkForAllInstructions(CheckForNoUnwind, *this, Opcodes,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- return ChangeStatus::UNCHANGED;
- }
-};
-
-struct AANoUnwindFunction final : public AANoUnwindImpl {
- AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
- : AANoUnwindImpl(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
-};
-
-/// NoUnwind attribute deduction for a call sites.
-struct AANoUnwindCallSite final
- : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
- AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
-};
-} // namespace
-
-/// ------------------------ NoSync Function Attribute -------------------------
-
-bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
- switch (CB.getIntrinsicID()) {
- case Intrinsic::nvvm_barrier0:
- case Intrinsic::nvvm_barrier0_and:
- case Intrinsic::nvvm_barrier0_or:
- case Intrinsic::nvvm_barrier0_popc:
- return true;
- case Intrinsic::amdgcn_s_barrier:
- if (ExecutedAligned)
- return true;
- break;
- default:
- break;
- }
- return hasAssumption(CB, KnownAssumptionString("ompx_aligned_barrier"));
-}
-
-bool AANoSync::isNonRelaxedAtomic(const Instruction *I) {
- if (!I->isAtomic())
- return false;
-
- if (auto *FI = dyn_cast<FenceInst>(I))
- // All legal orderings for fence are stronger than monotonic.
- return FI->getSyncScopeID() != SyncScope::SingleThread;
- if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
- // Unordered is not a legal ordering for cmpxchg.
- return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
- AI->getFailureOrdering() != AtomicOrdering::Monotonic);
- }
-
- AtomicOrdering Ordering;
- switch (I->getOpcode()) {
- case Instruction::AtomicRMW:
- Ordering = cast<AtomicRMWInst>(I)->getOrdering();
- break;
- case Instruction::Store:
- Ordering = cast<StoreInst>(I)->getOrdering();
- break;
- case Instruction::Load:
- Ordering = cast<LoadInst>(I)->getOrdering();
- break;
- default:
- llvm_unreachable(
- "New atomic operations need to be known in the attributor.");
- }
-
- return (Ordering != AtomicOrdering::Unordered &&
- Ordering != AtomicOrdering::Monotonic);
-}
-
-/// Return true if this intrinsic is nosync. This is only used for intrinsics
-/// which would be nosync except that they have a volatile flag. All other
-/// intrinsics are simply annotated with the nosync attribute in Intrinsics.td.
-bool AANoSync::isNoSyncIntrinsic(const Instruction *I) {
- if (auto *MI = dyn_cast<MemIntrinsic>(I))
- return !MI->isVolatile();
- return false;
-}
-
-namespace {
-struct AANoSyncImpl : AANoSync {
- AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- bool IsKnown;
- assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
- DepClassTy::NONE, IsKnown));
- (void)IsKnown;
- }
-
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "nosync" : "may-sync";
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override;
-};
-
-ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
-
- auto CheckRWInstForNoSync = [&](Instruction &I) {
- return AA::isNoSyncInst(A, I, *this);
- };
-
- auto CheckForNoSync = [&](Instruction &I) {
- // At this point we handled all read/write effects and they are all
- // nosync, so they can be skipped.
- if (I.mayReadOrWriteMemory())
- return true;
-
- bool IsKnown;
- CallBase &CB = cast<CallBase>(I);
- if (AA::hasAssumedIRAttr<Attribute::NoSync>(
- A, this, IRPosition::callsite_function(CB), DepClassTy::OPTIONAL,
- IsKnown))
- return true;
-
- // non-convergent and readnone imply nosync.
- return !CB.isConvergent();
- };
-
- bool UsedAssumedInformation = false;
- if (!A.checkForAllReadWriteInstructions(CheckRWInstForNoSync, *this,
- UsedAssumedInformation) ||
- !A.checkForAllCallLikeInstructions(CheckForNoSync, *this,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- return ChangeStatus::UNCHANGED;
-}
-
-struct AANoSyncFunction final : public AANoSyncImpl {
- AANoSyncFunction(const IRPosition &IRP, Attributor &A)
- : AANoSyncImpl(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
-};
-
-/// NoSync attribute deduction for a call sites.
-struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
- AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
-};
-} // namespace
-
-/// ------------------------ No-Free Attributes ----------------------------
-
-namespace {
-struct AANoFreeImpl : public AANoFree {
- AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- bool IsKnown;
- assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
- DepClassTy::NONE, IsKnown));
- (void)IsKnown;
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto CheckForNoFree = [&](Instruction &I) {
- bool IsKnown;
- return AA::hasAssumedIRAttr<Attribute::NoFree>(
- A, this, IRPosition::callsite_function(cast<CallBase>(I)),
- DepClassTy::REQUIRED, IsKnown);
- };
-
- bool UsedAssumedInformation = false;
- if (!A.checkForAllCallLikeInstructions(CheckForNoFree, *this,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "nofree" : "may-free";
- }
-};
-
-struct AANoFreeFunction final : public AANoFreeImpl {
- AANoFreeFunction(const IRPosition &IRP, Attributor &A)
- : AANoFreeImpl(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
-};
-
-/// NoFree attribute deduction for a call sites.
-struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
- AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
-};
-
-/// NoFree attribute for floating values.
-struct AANoFreeFloating : AANoFreeImpl {
- AANoFreeFloating(const IRPosition &IRP, Attributor &A)
- : AANoFreeImpl(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
-
- /// See Abstract Attribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- const IRPosition &IRP = getIRPosition();
-
- bool IsKnown;
- if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this,
- IRPosition::function_scope(IRP),
- DepClassTy::OPTIONAL, IsKnown))
- return ChangeStatus::UNCHANGED;
-
- Value &AssociatedValue = getIRPosition().getAssociatedValue();
- auto Pred = [&](const Use &U, bool &Follow) -> bool {
- Instruction *UserI = cast<Instruction>(U.getUser());
- if (auto *CB = dyn_cast<CallBase>(UserI)) {
- if (CB->isBundleOperand(&U))
- return false;
- if (!CB->isArgOperand(&U))
- return true;
- unsigned ArgNo = CB->getArgOperandNo(&U);
-
- bool IsKnown;
- return AA::hasAssumedIRAttr<Attribute::NoFree>(
- A, this, IRPosition::callsite_argument(*CB, ArgNo),
- DepClassTy::REQUIRED, IsKnown);
- }
-
- if (isa<GetElementPtrInst>(UserI) || isa<BitCastInst>(UserI) ||
- isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
- Follow = true;
- return true;
- }
- if (isa<StoreInst>(UserI) || isa<LoadInst>(UserI) ||
- isa<ReturnInst>(UserI))
- return true;
-
- // Unknown user.
- return false;
- };
- if (!A.checkForAllUses(Pred, *this, AssociatedValue))
- return indicatePessimisticFixpoint();
-
- return ChangeStatus::UNCHANGED;
- }
-};
-
-/// NoFree attribute for a call site argument.
-struct AANoFreeArgument final : AANoFreeFloating {
- AANoFreeArgument(const IRPosition &IRP, Attributor &A)
- : AANoFreeFloating(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
-};
-
-/// NoFree attribute for call site arguments.
-struct AANoFreeCallSiteArgument final : AANoFreeFloating {
- AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AANoFreeFloating(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // TODO: Once we have call site specific value information we can provide
- // call site specific liveness information and then it makes
- // sense to specialize attributes for call sites arguments instead of
- // redirecting requests to the callee argument.
- Argument *Arg = getAssociatedArgument();
- if (!Arg)
- return indicatePessimisticFixpoint();
- const IRPosition &ArgPos = IRPosition::argument(*Arg);
- bool IsKnown;
- if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this, ArgPos,
- DepClassTy::REQUIRED, IsKnown))
- return ChangeStatus::UNCHANGED;
- return indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nofree)};
-};
-
-/// NoFree attribute for function return value.
-struct AANoFreeReturned final : AANoFreeFloating {
- AANoFreeReturned(const IRPosition &IRP, Attributor &A)
- : AANoFreeFloating(IRP, A) {
- llvm_unreachable("NoFree is not applicable to function returns!");
- }
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- llvm_unreachable("NoFree is not applicable to function returns!");
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- llvm_unreachable("NoFree is not applicable to function returns!");
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {}
-};
-
-/// NoFree attribute deduction for a call site return value.
-struct AANoFreeCallSiteReturned final : AANoFreeFloating {
- AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AANoFreeFloating(IRP, A) {}
-
- ChangeStatus manifest(Attributor &A) override {
- return ChangeStatus::UNCHANGED;
- }
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
-};
-} // namespace
-
-/// ------------------------ NonNull Argument Attribute ------------------------
-
-bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions) {
- SmallVector<Attribute::AttrKind, 2> AttrKinds;
- AttrKinds.push_back(Attribute::NonNull);
- if (!NullPointerIsDefined(IRP.getAnchorScope(),
- IRP.getAssociatedType()->getPointerAddressSpace()))
- AttrKinds.push_back(Attribute::Dereferenceable);
- if (A.hasAttr(IRP, AttrKinds, IgnoreSubsumingPositions, Attribute::NonNull))
- return true;
-
- DominatorTree *DT = nullptr;
- AssumptionCache *AC = nullptr;
- InformationCache &InfoCache = A.getInfoCache();
- if (const Function *Fn = IRP.getAnchorScope()) {
- if (!Fn->isDeclaration()) {
- DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*Fn);
- AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*Fn);
- }
- }
-
- SmallVector<AA::ValueAndContext> Worklist;
- if (IRP.getPositionKind() != IRP_RETURNED) {
- Worklist.push_back({IRP.getAssociatedValue(), IRP.getCtxI()});
- } else {
- bool UsedAssumedInformation = false;
- if (!A.checkForAllInstructions(
- [&](Instruction &I) {
- Worklist.push_back({*cast<ReturnInst>(I).getReturnValue(), &I});
- return true;
- },
- IRP.getAssociatedFunction(), nullptr, {Instruction::Ret},
- UsedAssumedInformation))
- return false;
- }
-
- if (llvm::any_of(Worklist, [&](AA::ValueAndContext VAC) {
- return !isKnownNonZero(
- VAC.getValue(),
- SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
- }))
- return false;
-
- A.manifestAttrs(IRP, {Attribute::get(IRP.getAnchorValue().getContext(),
- Attribute::NonNull)});
- return true;
-}
-
-namespace {
-static int64_t getKnownNonNullAndDerefBytesForUse(
- Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
- const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
- TrackUse = false;
-
- const Value *UseV = U->get();
- if (!UseV->getType()->isPointerTy())
- return 0;
-
- // We need to follow common pointer manipulation uses to the accesses they
- // feed into. We can try to be smart to avoid looking through things we do not
- // like for now, e.g., non-inbounds GEPs.
- if (isa<CastInst>(I)) {
- TrackUse = true;
- return 0;
- }
-
- if (isa<GetElementPtrInst>(I)) {
- TrackUse = true;
- return 0;
- }
-
- Type *PtrTy = UseV->getType();
- const Function *F = I->getFunction();
- bool NullPointerIsDefined =
- F ? llvm::NullPointerIsDefined(F, PtrTy->getPointerAddressSpace()) : true;
- const DataLayout &DL = A.getInfoCache().getDL();
- if (const auto *CB = dyn_cast<CallBase>(I)) {
- if (CB->isBundleOperand(U)) {
- if (RetainedKnowledge RK = getKnowledgeFromUse(
- U, {Attribute::NonNull, Attribute::Dereferenceable})) {
- IsNonNull |=
- (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);
- return RK.ArgValue;
- }
- return 0;
- }
-
- if (CB->isCallee(U)) {
- IsNonNull |= !NullPointerIsDefined;
- return 0;
- }
-
- unsigned ArgNo = CB->getArgOperandNo(U);
- IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
- // As long as we only use known information there is no need to track
- // dependences here.
- bool IsKnownNonNull;
- AA::hasAssumedIRAttr<Attribute::NonNull>(A, &QueryingAA, IRP,
- DepClassTy::NONE, IsKnownNonNull);
- IsNonNull |= IsKnownNonNull;
- auto *DerefAA =
- A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClassTy::NONE);
- return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
- }
-
- std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);
- if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
- Loc->Size.isScalable() || I->isVolatile())
- return 0;
-
- int64_t Offset;
- const Value *Base =
- getMinimalBaseOfPointer(A, QueryingAA, Loc->Ptr, Offset, DL);
- if (Base && Base == &AssociatedValue) {
- int64_t DerefBytes = Loc->Size.getValue() + Offset;
- IsNonNull |= !NullPointerIsDefined;
- return std::max(int64_t(0), DerefBytes);
- }
-
- /// Corner case when an offset is 0.
- Base = GetPointerBaseWithConstantOffset(Loc->Ptr, Offset, DL,
- /*AllowNonInbounds*/ true);
- if (Base && Base == &AssociatedValue && Offset == 0) {
- int64_t DerefBytes = Loc->Size.getValue();
- IsNonNull |= !NullPointerIsDefined;
- return std::max(int64_t(0), DerefBytes);
- }
-
- return 0;
-}
-
-struct AANonNullImpl : AANonNull {
- AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- Value &V = *getAssociatedValue().stripPointerCasts();
- if (isa<ConstantPointerNull>(V)) {
- indicatePessimisticFixpoint();
- return;
- }
-
- if (Instruction *CtxI = getCtxI())
- followUsesInMBEC(*this, A, getState(), *CtxI);
- }
-
- /// See followUsesInMBEC
- bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
- AANonNull::StateType &State) {
- bool IsNonNull = false;
- bool TrackUse = false;
- getKnownNonNullAndDerefBytesForUse(A, *this, getAssociatedValue(), U, I,
- IsNonNull, TrackUse);
- State.setKnown(IsNonNull);
- return TrackUse;
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "nonnull" : "may-null";
- }
-};
-
-/// NonNull attribute for a floating value.
-struct AANonNullFloating : public AANonNullImpl {
- AANonNullFloating(const IRPosition &IRP, Attributor &A)
- : AANonNullImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto CheckIRP = [&](const IRPosition &IRP) {
- bool IsKnownNonNull;
- return AA::hasAssumedIRAttr<Attribute::NonNull>(
- A, *this, IRP, DepClassTy::OPTIONAL, IsKnownNonNull);
- };
-
- bool Stripped;
- bool UsedAssumedInformation = false;
- Value *AssociatedValue = &getAssociatedValue();
- SmallVector<AA::ValueAndContext> Values;
- if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
- AA::AnyScope, UsedAssumedInformation))
- Stripped = false;
- else
- Stripped =
- Values.size() != 1 || Values.front().getValue() != AssociatedValue;
-
- if (!Stripped) {
- bool IsKnown;
- if (auto *PHI = dyn_cast<PHINode>(AssociatedValue))
- if (llvm::all_of(PHI->incoming_values(), [&](Value *Op) {
- return AA::hasAssumedIRAttr<Attribute::NonNull>(
- A, this, IRPosition::value(*Op), DepClassTy::OPTIONAL,
- IsKnown);
- }))
- return ChangeStatus::UNCHANGED;
- if (auto *Select = dyn_cast<SelectInst>(AssociatedValue))
- if (AA::hasAssumedIRAttr<Attribute::NonNull>(
- A, this, IRPosition::value(*Select->getFalseValue()),
- DepClassTy::OPTIONAL, IsKnown) &&
- AA::hasAssumedIRAttr<Attribute::NonNull>(
- A, this, IRPosition::value(*Select->getTrueValue()),
- DepClassTy::OPTIONAL, IsKnown))
- return ChangeStatus::UNCHANGED;
-
- // If we haven't stripped anything we might still be able to use a
- // different AA, but only if the IRP changes. Effectively when we
- // interpret this not as a call site value but as a floating/argument
- // value.
- const IRPosition AVIRP = IRPosition::value(*AssociatedValue);
- if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
- return indicatePessimisticFixpoint();
- return ChangeStatus::UNCHANGED;
- }
-
- for (const auto &VAC : Values)
- if (!CheckIRP(IRPosition::value(*VAC.getValue())))
- return indicatePessimisticFixpoint();
-
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
-};
-
-/// NonNull attribute for function return value.
-struct AANonNullReturned final
- : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
- false, AANonNull::IRAttributeKind, false> {
- AANonNullReturned(const IRPosition &IRP, Attributor &A)
- : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
- false, Attribute::NonNull, false>(IRP, A) {
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "nonnull" : "may-null";
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
-};
-
-/// NonNull attribute for function argument.
-struct AANonNullArgument final
- : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
- AANonNullArgument(const IRPosition &IRP, Attributor &A)
- : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
-};
-
-struct AANonNullCallSiteArgument final : AANonNullFloating {
- AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AANonNullFloating(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
-};
-
-/// NonNull attribute for a call site return position.
-struct AANonNullCallSiteReturned final
- : AACalleeToCallSite<AANonNull, AANonNullImpl> {
- AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
-};
-} // namespace
-
-/// ------------------------ Must-Progress Attributes --------------------------
-namespace {
-struct AAMustProgressImpl : public AAMustProgress {
- AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
- : AAMustProgress(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- bool IsKnown;
- assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
- A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
- (void)IsKnown;
- }
-
- /// See AbstractAttribute::getAsStr()
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "mustprogress" : "may-not-progress";
- }
-};
-
-struct AAMustProgressFunction final : AAMustProgressImpl {
- AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
- : AAMustProgressImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- bool IsKnown;
- if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
- A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnown)) {
- if (IsKnown)
- return indicateOptimisticFixpoint();
- return ChangeStatus::UNCHANGED;
- }
-
- auto CheckForMustProgress = [&](AbstractCallSite ACS) {
- IRPosition IPos = IRPosition::callsite_function(*ACS.getInstruction());
- bool IsKnownMustProgress;
- return AA::hasAssumedIRAttr<Attribute::MustProgress>(
- A, this, IPos, DepClassTy::REQUIRED, IsKnownMustProgress,
- /* IgnoreSubsumingPositions */ true);
- };
-
- bool AllCallSitesKnown = true;
- if (!A.checkForAllCallSites(CheckForMustProgress, *this,
- /* RequireAllCallSites */ true,
- AllCallSitesKnown))
- return indicatePessimisticFixpoint();
-
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FN_ATTR(mustprogress)
- }
-};
-
-/// MustProgress attribute deduction for a call sites.
-struct AAMustProgressCallSite final : AAMustProgressImpl {
- AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
- : AAMustProgressImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // TODO: Once we have call site specific value information we can provide
- // call site specific liveness information and then it makes
- // sense to specialize attributes for call sites arguments instead of
- // redirecting requests to the callee argument.
- const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
- bool IsKnownMustProgress;
- if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
- A, this, FnPos, DepClassTy::REQUIRED, IsKnownMustProgress))
- return indicatePessimisticFixpoint();
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CS_ATTR(mustprogress);
- }
-};
-} // namespace
-
-/// ------------------------ No-Recurse Attributes ----------------------------
-
-namespace {
-struct AANoRecurseImpl : public AANoRecurse {
- AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- bool IsKnown;
- assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
- A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
- (void)IsKnown;
- }
-
- /// See AbstractAttribute::getAsStr()
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "norecurse" : "may-recurse";
- }
-};
-
-struct AANoRecurseFunction final : AANoRecurseImpl {
- AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
- : AANoRecurseImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
-
- // If all live call sites are known to be no-recurse, we are as well.
- auto CallSitePred = [&](AbstractCallSite ACS) {
- bool IsKnownNoRecurse;
- if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
- A, this,
- IRPosition::function(*ACS.getInstruction()->getFunction()),
- DepClassTy::NONE, IsKnownNoRecurse))
- return false;
- return IsKnownNoRecurse;
- };
- bool UsedAssumedInformation = false;
- if (A.checkForAllCallSites(CallSitePred, *this, true,
- UsedAssumedInformation)) {
- // If we know all call sites and all are known no-recurse, we are done.
- // If all known call sites, which might not be all that exist, are known
- // to be no-recurse, we are not done but we can continue to assume
- // no-recurse. If one of the call sites we have not visited will become
- // live, another update is triggered.
- if (!UsedAssumedInformation)
- indicateOptimisticFixpoint();
- return ChangeStatus::UNCHANGED;
- }
-
- const AAInterFnReachability *EdgeReachability =
- A.getAAFor<AAInterFnReachability>(*this, getIRPosition(),
- DepClassTy::REQUIRED);
- if (EdgeReachability && EdgeReachability->canReach(A, *getAnchorScope()))
- return indicatePessimisticFixpoint();
- return ChangeStatus::UNCHANGED;
- }
-
- void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
-};
-
-/// NoRecurse attribute deduction for a call sites.
-struct AANoRecurseCallSite final
- : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
- AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
-};
-} // namespace
-
-/// ------------------------ No-Convergent Attribute --------------------------
-
-namespace {
-struct AANonConvergentImpl : public AANonConvergent {
- AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
- : AANonConvergent(IRP, A) {}
-
- /// See AbstractAttribute::getAsStr()
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "non-convergent" : "may-be-convergent";
- }
-};
-
-struct AANonConvergentFunction final : AANonConvergentImpl {
- AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
- : AANonConvergentImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // If all function calls are known to not be convergent, we are not
- // convergent.
- auto CalleeIsNotConvergent = [&](Instruction &Inst) {
- CallBase &CB = cast<CallBase>(Inst);
- auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand());
- if (!Callee || Callee->isIntrinsic()) {
- return false;
- }
- if (Callee->isDeclaration()) {
- return !Callee->hasFnAttribute(Attribute::Convergent);
- }
- const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
- *this, IRPosition::function(*Callee), DepClassTy::REQUIRED);
- return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
- };
-
- bool UsedAssumedInformation = false;
- if (!A.checkForAllCallLikeInstructions(CalleeIsNotConvergent, *this,
- UsedAssumedInformation)) {
- return indicatePessimisticFixpoint();
- }
- return ChangeStatus::UNCHANGED;
- }
-
- ChangeStatus manifest(Attributor &A) override {
- if (isKnownNotConvergent() &&
- A.hasAttr(getIRPosition(), Attribute::Convergent)) {
- A.removeAttrs(getIRPosition(), {Attribute::Convergent});
- return ChangeStatus::CHANGED;
- }
- return ChangeStatus::UNCHANGED;
- }
-
- void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
-};
-} // namespace
-
-/// -------------------- Undefined-Behavior Attributes ------------------------
-
-namespace {
-struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
- AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
- : AAUndefinedBehavior(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- // through a pointer (i.e. also branches etc.)
- ChangeStatus updateImpl(Attributor &A) override {
- const size_t UBPrevSize = KnownUBInsts.size();
- const size_t NoUBPrevSize = AssumedNoUBInsts.size();
-
- auto InspectMemAccessInstForUB = [&](Instruction &I) {
- // Lang ref now states volatile store is not UB, let's skip them.
- if (I.isVolatile() && I.mayWriteToMemory())
- return true;
-
- // Skip instructions that are already saved.
- if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
- return true;
-
- // If we reach here, we know we have an instruction
- // that accesses memory through a pointer operand,
- // for which getPointerOperand() should give it to us.
- Value *PtrOp =
- const_cast<Value *>(getPointerOperand(&I, /* AllowVolatile */ true));
- assert(PtrOp &&
- "Expected pointer operand of memory accessing instruction");
-
- // Either we stopped and the appropriate action was taken,
- // or we got back a simplified value to continue.
- std::optional<Value *> SimplifiedPtrOp =
- stopOnUndefOrAssumed(A, PtrOp, &I);
- if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
- return true;
- const Value *PtrOpVal = *SimplifiedPtrOp;
-
- // A memory access through a pointer is considered UB
- // only if the pointer has constant null value.
- // TODO: Expand it to not only check constant values.
- if (!isa<ConstantPointerNull>(PtrOpVal)) {
- AssumedNoUBInsts.insert(&I);
- return true;
- }
- const Type *PtrTy = PtrOpVal->getType();
-
- // Because we only consider instructions inside functions,
- // assume that a parent function exists.
- const Function *F = I.getFunction();
-
- // A memory access using constant null pointer is only considered UB
- // if null pointer is _not_ defined for the target platform.
- if (llvm::NullPointerIsDefined(F, PtrTy->getPointerAddressSpace()))
- AssumedNoUBInsts.insert(&I);
- else
- KnownUBInsts.insert(&I);
- return true;
- };
-
- auto InspectBrInstForUB = [&](Instruction &I) {
- // A conditional branch instruction is considered UB if it has `undef`
- // condition.
-
- // Skip instructions that are already saved.
- if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
- return true;
-
- // We know we have a branch instruction.
- auto *BrInst = cast<BranchInst>(&I);
-
- // Unconditional branches are never considered UB.
- if (BrInst->isUnconditional())
- return true;
-
- // Either we stopped and the appropriate action was taken,
- // or we got back a simplified value to continue.
- std::optional<Value *> SimplifiedCond =
- stopOnUndefOrAssumed(A, BrInst->getCondition(), BrInst);
- if (!SimplifiedCond || !*SimplifiedCond)
- return true;
- AssumedNoUBInsts.insert(&I);
- return true;
- };
-
- auto InspectCallSiteForUB = [&](Instruction &I) {
- // Check whether a callsite always cause UB or not
-
- // Skip instructions that are already saved.
- if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
- return true;
-
- // Check nonnull and noundef argument attribute violation for each
- // callsite.
- CallBase &CB = cast<CallBase>(I);
- auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand());
- if (!Callee)
- return true;
- for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
- // If current argument is known to be simplified to null pointer and the
- // corresponding argument position is known to have nonnull attribute,
- // the argument is poison. Furthermore, if the argument is poison and
- // the position is known to have noundef attriubte, this callsite is
- // considered UB.
- if (idx >= Callee->arg_size())
- break;
- Value *ArgVal = CB.getArgOperand(idx);
- if (!ArgVal)
- continue;
- // Here, we handle three cases.
- // (1) Not having a value means it is dead. (we can replace the value
- // with undef)
- // (2) Simplified to undef. The argument violate noundef attriubte.
- // (3) Simplified to null pointer where known to be nonnull.
- // The argument is a poison value and violate noundef attribute.
- IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, idx);
- bool IsKnownNoUndef;
- AA::hasAssumedIRAttr<Attribute::NoUndef>(
- A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNoUndef);
- if (!IsKnownNoUndef)
- continue;
- bool UsedAssumedInformation = false;
- std::optional<Value *> SimplifiedVal =
- A.getAssumedSimplified(IRPosition::value(*ArgVal), *this,
- UsedAssumedInformation, AA::Interprocedural);
- if (UsedAssumedInformation)
- continue;
- if (SimplifiedVal && !*SimplifiedVal)
- return true;
- if (!SimplifiedVal || isa<UndefValue>(**SimplifiedVal)) {
- KnownUBInsts.insert(&I);
- continue;
- }
- if (!ArgVal->getType()->isPointerTy() ||
- !isa<ConstantPointerNull>(**SimplifiedVal))
- continue;
- bool IsKnownNonNull;
- AA::hasAssumedIRAttr<Attribute::NonNull>(
- A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNonNull);
- if (IsKnownNonNull)
- KnownUBInsts.insert(&I);
- }
- return true;
- };
-
- auto InspectReturnInstForUB = [&](Instruction &I) {
- auto &RI = cast<ReturnInst>(I);
- // Either we stopped and the appropriate action was taken,
- // or we got back a simplified return value to continue.
- std::optional<Value *> SimplifiedRetValue =
- stopOnUndefOrAssumed(A, RI.getReturnValue(), &I);
- if (!SimplifiedRetValue || !*SimplifiedRetValue)
- return true;
-
- // Check if a return instruction always cause UB or not
- // Note: It is guaranteed that the returned position of the anchor
- // scope has noundef attribute when this is called.
- // We also ensure the return position is not "assumed dead"
- // because the returned value was then potentially simplified to
- // `undef` in AAReturnedValues without removing the `noundef`
- // attribute yet.
-
- // When the returned position has noundef attriubte, UB occurs in the
- // following cases.
- // (1) Returned value is known to be undef.
- // (2) The value is known to be a null pointer and the returned
- // position has nonnull attribute (because the returned value is
- // poison).
- if (isa<ConstantPointerNull>(*SimplifiedRetValue)) {
- bool IsKnownNonNull;
- AA::hasAssumedIRAttr<Attribute::NonNull>(
- A, this, IRPosition::returned(*getAnchorScope()), DepClassTy::NONE,
- IsKnownNonNull);
- if (IsKnownNonNull)
- KnownUBInsts.insert(&I);
- }
-
- return true;
- };
-
- bool UsedAssumedInformation = false;
- A.checkForAllInstructions(InspectMemAccessInstForUB, *this,
- {Instruction::Load, Instruction::Store,
- Instruction::AtomicCmpXchg,
- Instruction::AtomicRMW},
- UsedAssumedInformation,
- /* CheckBBLivenessOnly */ true);
- A.checkForAllInstructions(InspectBrInstForUB, *this, {Instruction::Br},
- UsedAssumedInformation,
- /* CheckBBLivenessOnly */ true);
- A.checkForAllCallLikeInstructions(InspectCallSiteForUB, *this,
- UsedAssumedInformation);
-
- // If the returned position of the anchor scope has noundef attriubte, check
- // all returned instructions.
- if (!getAnchorScope()->getReturnType()->isVoidTy()) {
- const IRPosition &ReturnIRP = IRPosition::returned(*getAnchorScope());
- if (!A.isAssumedDead(ReturnIRP, this, nullptr, UsedAssumedInformation)) {
- bool IsKnownNoUndef;
- AA::hasAssumedIRAttr<Attribute::NoUndef>(
- A, this, ReturnIRP, DepClassTy::NONE, IsKnownNoUndef);
- if (IsKnownNoUndef)
- A.checkForAllInstructions(InspectReturnInstForUB, *this,
- {Instruction::Ret}, UsedAssumedInformation,
- /* CheckBBLivenessOnly */ true);
- }
- }
-
- if (NoUBPrevSize != AssumedNoUBInsts.size() ||
- UBPrevSize != KnownUBInsts.size())
- return ChangeStatus::CHANGED;
- return ChangeStatus::UNCHANGED;
- }
-
- bool isKnownToCauseUB(Instruction *I) const override {
- return KnownUBInsts.count(I);
- }
-
- bool isAssumedToCauseUB(Instruction *I) const override {
- // In simple words, if an instruction is not in the assumed to _not_
- // cause UB, then it is assumed UB (that includes those
- // in the KnownUBInsts set). The rest is boilerplate
- // is to ensure that it is one of the instructions we test
- // for UB.
-
- switch (I->getOpcode()) {
- case Instruction::Load:
- case Instruction::Store:
- case Instruction::AtomicCmpXchg:
- case Instruction::AtomicRMW:
- return !AssumedNoUBInsts.count(I);
- case Instruction::Br: {
- auto *BrInst = cast<BranchInst>(I);
- if (BrInst->isUnconditional())
- return false;
- return !AssumedNoUBInsts.count(I);
- } break;
- default:
- return false;
- }
- return false;
- }
-
- ChangeStatus manifest(Attributor &A) override {
- if (KnownUBInsts.empty())
- return ChangeStatus::UNCHANGED;
- for (Instruction *I : KnownUBInsts)
- A.changeToUnreachableAfterManifest(I);
- return ChangeStatus::CHANGED;
- }
-
- /// See AbstractAttribute::getAsStr()
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "undefined-behavior" : "no-ub";
- }
-
- /// Note: The correctness of this analysis depends on the fact that the
- /// following 2 sets will stop changing after some point.
- /// "Change" here means that their size changes.
- /// The size of each set is monotonically increasing
- /// (we only add items to them) and it is upper bounded by the number of
- /// instructions in the processed function (we can never save more
- /// elements in either set than this number). Hence, at some point,
- /// they will stop increasing.
- /// Consequently, at some point, both sets will have stopped
- /// changing, effectively making the analysis reach a fixpoint.
-
- /// Note: These 2 sets are disjoint and an instruction can be considered
- /// one of 3 things:
- /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
- /// the KnownUBInsts set.
- /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
- /// has a reason to assume it).
- /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
- /// could not find a reason to assume or prove that it can cause UB,
- /// hence it assumes it doesn't. We have a set for these instructions
- /// so that we don't reprocess them in every update.
- /// Note however that instructions in this set may cause UB.
-
-protected:
- /// A set of all live instructions _known_ to cause UB.
- SmallPtrSet<Instruction *, 8> KnownUBInsts;
-
-private:
- /// A set of all the (live) instructions that are assumed to _not_ cause UB.
- SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
-
- // Should be called on updates in which if we're processing an instruction
- // \p I that depends on a value \p V, one of the following has to happen:
- // - If the value is assumed, then stop.
- // - If the value is known but undef, then consider it UB.
- // - Otherwise, do specific processing with the simplified value.
- // We return std::nullopt in the first 2 cases to signify that an appropriate
- // action was taken and the caller should stop.
- // Otherwise, we return the simplified value that the caller should
- // use for specific processing.
- std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
- Instruction *I) {
- bool UsedAssumedInformation = false;
- std::optional<Value *> SimplifiedV =
- A.getAssumedSimplified(IRPosition::value(*V), *this,
- UsedAssumedInformation, AA::Interprocedural);
- if (!UsedAssumedInformation) {
- // Don't depend on assumed values.
- if (!SimplifiedV) {
- // If it is known (which we tested above) but it doesn't have a value,
- // then we can assume `undef` and hence the instruction is UB.
- KnownUBInsts.insert(I);
- return std::nullopt;
- }
- if (!*SimplifiedV)
- return nullptr;
- V = *SimplifiedV;
- }
- if (isa<UndefValue>(V)) {
- KnownUBInsts.insert(I);
- return std::nullopt;
- }
- return V;
- }
-};
-
-struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
- AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
- : AAUndefinedBehaviorImpl(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECL(UndefinedBehaviorInstruction, Instruction,
- "Number of instructions known to have UB");
- BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
- KnownUBInsts.size();
- }
-};
-} // namespace
-
-/// ------------------------ Will-Return Attributes ----------------------------
-
-namespace {
-// Helper function that checks whether a function has any cycle which we don't
-// know if it is bounded or not.
-// Loops with maximum trip count are considered bounded, any other cycle not.
-static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
- ScalarEvolution *SE =
- A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
- LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
- // If either SCEV or LoopInfo is not available for the function then we assume
- // any cycle to be unbounded cycle.
- // We use scc_iterator which uses Tarjan algorithm to find all the maximal
- // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
- if (!SE || !LI) {
- for (scc_iterator<Function *> SCCI = scc_begin(&F); !SCCI.isAtEnd(); ++SCCI)
- if (SCCI.hasCycle())
- return true;
- return false;
- }
-
- // If there's irreducible control, the function may contain non-loop cycles.
- if (mayContainIrreducibleControl(F, LI))
- return true;
-
- // Any loop that does not have a max trip count is considered unbounded cycle.
- for (auto *L : LI->getLoopsInPreorder()) {
- if (!SE->getSmallConstantMaxTripCount(L))
- return true;
- }
- return false;
-}
-
-struct AAWillReturnImpl : public AAWillReturn {
- AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
- : AAWillReturn(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- bool IsKnown;
- assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
- A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
- (void)IsKnown;
- }
-
- /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
- bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
- if (!A.hasAttr(getIRPosition(), {Attribute::MustProgress}))
- return false;
-
- bool IsKnown;
- if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
- return IsKnown || !KnownOnly;
- return false;
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
- return ChangeStatus::UNCHANGED;
-
- auto CheckForWillReturn = [&](Instruction &I) {
- IRPosition IPos = IRPosition::callsite_function(cast<CallBase>(I));
- bool IsKnown;
- if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
- A, this, IPos, DepClassTy::REQUIRED, IsKnown)) {
- if (IsKnown)
- return true;
- } else {
- return false;
- }
- bool IsKnownNoRecurse;
- return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
- A, this, IPos, DepClassTy::REQUIRED, IsKnownNoRecurse);
- };
-
- bool UsedAssumedInformation = false;
- if (!A.checkForAllCallLikeInstructions(CheckForWillReturn, *this,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::getAsStr()
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "willreturn" : "may-noreturn";
- }
-};
-
-struct AAWillReturnFunction final : AAWillReturnImpl {
- AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
- : AAWillReturnImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- AAWillReturnImpl::initialize(A);
-
- Function *F = getAnchorScope();
- assert(F && "Did expect an anchor function");
- if (F->isDeclaration() || mayContainUnboundedCycle(*F, A))
- indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
-};
-
-/// WillReturn attribute deduction for a call sites.
-struct AAWillReturnCallSite final
- : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
- AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
- return ChangeStatus::UNCHANGED;
-
- return AACalleeToCallSite::updateImpl(A);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
-};
-} // namespace
-
-/// -------------------AAIntraFnReachability Attribute--------------------------
-
-/// All information associated with a reachability query. This boilerplate code
-/// is used by both AAIntraFnReachability and AAInterFnReachability, with
-/// different \p ToTy values.
-template <typename ToTy> struct ReachabilityQueryInfo {
- enum class Reachable {
- No,
- Yes,
- };
-
- /// Start here,
- const Instruction *From = nullptr;
- /// reach this place,
- const ToTy *To = nullptr;
- /// without going through any of these instructions,
- const AA::InstExclusionSetTy *ExclusionSet = nullptr;
- /// and remember if it worked:
- Reachable Result = Reachable::No;
-
- /// Precomputed hash for this RQI.
- unsigned Hash = 0;
-
- unsigned computeHashValue() const {
- assert(Hash == 0 && "Computed hash twice!");
- using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
- using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
- return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
- detail::combineHashValue(PairDMI ::getHashValue({From, To}),
- InstSetDMI::getHashValue(ExclusionSet));
- }
-
- ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
- : From(From), To(To) {}
-
- /// Constructor replacement to ensure unique and stable sets are used for the
- /// cache.
- ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To,
- const AA::InstExclusionSetTy *ES, bool MakeUnique)
- : From(&From), To(&To), ExclusionSet(ES) {
-
- if (!ES || ES->empty()) {
- ExclusionSet = nullptr;
- } else if (MakeUnique) {
- ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(ES);
- }
- }
-
- ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
- : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
-};
-
-namespace llvm {
-template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
- using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
- using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
-
- static ReachabilityQueryInfo<ToTy> EmptyKey;
- static ReachabilityQueryInfo<ToTy> TombstoneKey;
-
- static inline ReachabilityQueryInfo<ToTy> *getEmptyKey() { return &EmptyKey; }
- static inline ReachabilityQueryInfo<ToTy> *getTombstoneKey() {
- return &TombstoneKey;
- }
- static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
- return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
- }
- static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,
- const ReachabilityQueryInfo<ToTy> *RHS) {
- if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
- return false;
- return InstSetDMI::isEqual(LHS->ExclusionSet, RHS->ExclusionSet);
- }
-};
-
-#define DefineKeys(ToTy) \
- template <> \
- ReachabilityQueryInfo<ToTy> \
- DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::EmptyKey = \
- ReachabilityQueryInfo<ToTy>( \
- DenseMapInfo<const Instruction *>::getEmptyKey(), \
- DenseMapInfo<const ToTy *>::getEmptyKey()); \
- template <> \
- ReachabilityQueryInfo<ToTy> \
- DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::TombstoneKey = \
- ReachabilityQueryInfo<ToTy>( \
- DenseMapInfo<const Instruction *>::getTombstoneKey(), \
- DenseMapInfo<const ToTy *>::getTombstoneKey());
-
-DefineKeys(Instruction) DefineKeys(Function)
-#undef DefineKeys
-
-} // namespace llvm
-
-namespace {
-
-template <typename BaseTy, typename ToTy>
-struct CachedReachabilityAA : public BaseTy {
- using RQITy = ReachabilityQueryInfo<ToTy>;
-
- CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
-
- /// See AbstractAttribute::isQueryAA.
- bool isQueryAA() const override { return true; }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
- RQITy *RQI = QueryVector[u];
- if (RQI->Result == RQITy::Reachable::No &&
- isReachableImpl(A, *RQI, /*IsTemporaryRQI=*/false))
- Changed = ChangeStatus::CHANGED;
- }
- return Changed;
- }
-
- virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
- bool IsTemporaryRQI) = 0;
-
- bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
- RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
- RQI.Result = Result;
-
- // Remove the temporary RQI from the cache.
- if (IsTemporaryRQI)
- QueryCache.erase(&RQI);
-
- // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
- // 1) If it is reachable, it doesn't matter if we have an exclusion set for
- // this query. 2) We did not use the exclusion set, potentially because
- // there is none.
- if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
- RQITy PlainRQI(RQI.From, RQI.To);
- if (!QueryCache.count(&PlainRQI)) {
- RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
- RQIPtr->Result = Result;
- QueryVector.push_back(RQIPtr);
- QueryCache.insert(RQIPtr);
- }
- }
-
- // Check if we need to insert a new permanent RQI with the exclusion set.
- if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
- assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
- "Did not expect empty set!");
- RQITy *RQIPtr = new (A.Allocator)
- RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
- assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
- RQIPtr->Result = Result;
- assert(!QueryCache.count(RQIPtr));
- QueryVector.push_back(RQIPtr);
- QueryCache.insert(RQIPtr);
- }
-
- if (Result == RQITy::Reachable::No && IsTemporaryRQI)
- A.registerForUpdate(*this);
- return Result == RQITy::Reachable::Yes;
- }
-
- const std::string getAsStr(Attributor *A) const override {
- // TODO: Return the number of reachable queries.
- return "#queries(" + std::to_string(QueryVector.size()) + ")";
- }
-
- bool checkQueryCache(Attributor &A, RQITy &StackRQI,
- typename RQITy::Reachable &Result) {
- if (!this->getState().isValidState()) {
- Result = RQITy::Reachable::Yes;
- return true;
- }
-
- // If we have an exclusion set we might be able to find our answer by
- // ignoring it first.
- if (StackRQI.ExclusionSet) {
- RQITy PlainRQI(StackRQI.From, StackRQI.To);
- auto It = QueryCache.find(&PlainRQI);
- if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
- Result = RQITy::Reachable::No;
- return true;
- }
- }
-
- auto It = QueryCache.find(&StackRQI);
- if (It != QueryCache.end()) {
- Result = (*It)->Result;
- return true;
- }
-
- // Insert a temporary for recursive queries. We will replace it with a
- // permanent entry later.
- QueryCache.insert(&StackRQI);
- return false;
- }
-
-private:
- SmallVector<RQITy *> QueryVector;
- DenseSet<RQITy *> QueryCache;
-};
-
-struct AAIntraFnReachabilityFunction final
- : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
- using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
- AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {
- DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
- *IRP.getAssociatedFunction());
- }
-
- bool isAssumedReachable(
- Attributor &A, const Instruction &From, const Instruction &To,
- const AA::InstExclusionSetTy *ExclusionSet) const override {
- auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
- if (&From == &To)
- return true;
-
- RQITy StackRQI(A, From, To, ExclusionSet, false);
- typename RQITy::Reachable Result;
- if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
- return NonConstThis->isReachableImpl(A, StackRQI,
- /*IsTemporaryRQI=*/true);
- return Result == RQITy::Reachable::Yes;
- }
-
- ChangeStatus updateImpl(Attributor &A) override {
- // We only depend on liveness. DeadEdges is all we care about, check if any
- // of them changed.
- auto *LivenessAA =
- A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
- if (LivenessAA &&
- llvm::all_of(DeadEdges,
- [&](const auto &DeadEdge) {
- return LivenessAA->isEdgeDead(DeadEdge.first,
- DeadEdge.second);
- }) &&
- llvm::all_of(DeadBlocks, [&](const BasicBlock *BB) {
- return LivenessAA->isAssumedDead(BB);
- })) {
- return ChangeStatus::UNCHANGED;
- }
- DeadEdges.clear();
- DeadBlocks.clear();
- return Base::updateImpl(A);
- }
-
- bool isReachableImpl(Attributor &A, RQITy &RQI,
- bool IsTemporaryRQI) override {
- const Instruction *Origin = RQI.From;
- bool UsedExclusionSet = false;
-
- auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
- const AA::InstExclusionSetTy *ExclusionSet) {
- const Instruction *IP = &From;
- while (IP && IP != &To) {
- if (ExclusionSet && IP != Origin && ExclusionSet->count(IP)) {
- UsedExclusionSet = true;
- break;
- }
- IP = IP->getNextNode();
- }
- return IP == &To;
- };
-
- const BasicBlock *FromBB = RQI.From->getParent();
- const BasicBlock *ToBB = RQI.To->getParent();
- assert(FromBB->getParent() == ToBB->getParent() &&
- "Not an intra-procedural query!");
-
- // Check intra-block reachability, however, other reaching paths are still
- // possible.
- if (FromBB == ToBB &&
- WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
- return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
- IsTemporaryRQI);
-
- // Check if reaching the ToBB block is sufficient or if even that would not
- // ensure reaching the target. In the latter case we are done.
- if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
- return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
- IsTemporaryRQI);
-
- const Function *Fn = FromBB->getParent();
- SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
- if (RQI.ExclusionSet)
- for (auto *I : *RQI.ExclusionSet)
- if (I->getFunction() == Fn)
- ExclusionBlocks.insert(I->getParent());
-
- // Check if we make it out of the FromBB block at all.
- if (ExclusionBlocks.count(FromBB) &&
- !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
- RQI.ExclusionSet))
- return rememberResult(A, RQITy::Reachable::No, RQI, true, IsTemporaryRQI);
-
- auto *LivenessAA =
- A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
- if (LivenessAA && LivenessAA->isAssumedDead(ToBB)) {
- DeadBlocks.insert(ToBB);
- return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
- IsTemporaryRQI);
- }
-
- SmallPtrSet<const BasicBlock *, 16> Visited;
- SmallVector<const BasicBlock *, 16> Worklist;
- Worklist.push_back(FromBB);
-
- DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
- while (!Worklist.empty()) {
- const BasicBlock *BB = Worklist.pop_back_val();
- if (!Visited.insert(BB).second)
- continue;
- for (const BasicBlock *SuccBB : successors(BB)) {
- if (LivenessAA && LivenessAA->isEdgeDead(BB, SuccBB)) {
- LocalDeadEdges.insert({BB, SuccBB});
- continue;
- }
- // We checked before if we just need to reach the ToBB block.
- if (SuccBB == ToBB)
- return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
- IsTemporaryRQI);
- if (DT && ExclusionBlocks.empty() && DT->dominates(BB, ToBB))
- return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
- IsTemporaryRQI);
-
- if (ExclusionBlocks.count(SuccBB)) {
- UsedExclusionSet = true;
- continue;
- }
- Worklist.push_back(SuccBB);
- }
- }
-
- DeadEdges.insert(LocalDeadEdges.begin(), LocalDeadEdges.end());
- return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
- IsTemporaryRQI);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {}
-
-private:
- // Set of assumed dead blocks we used in the last query. If any changes we
- // update the state.
- DenseSet<const BasicBlock *> DeadBlocks;
-
- // Set of assumed dead edges we used in the last query. If any changes we
- // update the state.
- DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
-
- /// The dominator tree of the function to short-circuit reasoning.
- const DominatorTree *DT = nullptr;
-};
-} // namespace
-
-/// ------------------------ NoAlias Argument Attribute ------------------------
-
-bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions) {
- assert(ImpliedAttributeKind == Attribute::NoAlias &&
- "Unexpected attribute kind");
- Value *Val = &IRP.getAssociatedValue();
- if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
- if (isa<AllocaInst>(Val))
- return true;
- } else {
- IgnoreSubsumingPositions = true;
- }
-
- if (isa<UndefValue>(Val))
- return true;
-
- if (isa<ConstantPointerNull>(Val) &&
- !NullPointerIsDefined(IRP.getAnchorScope(),
- Val->getType()->getPointerAddressSpace()))
- return true;
-
- if (A.hasAttr(IRP, {Attribute::ByVal, Attribute::NoAlias},
- IgnoreSubsumingPositions, Attribute::NoAlias))
- return true;
-
- return false;
-}
-
-namespace {
-struct AANoAliasImpl : AANoAlias {
- AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
- assert(getAssociatedType()->isPointerTy() &&
- "Noalias is a pointer attribute");
- }
-
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "noalias" : "may-alias";
- }
-};
-
-/// NoAlias attribute for a floating value.
-struct AANoAliasFloating final : AANoAliasImpl {
- AANoAliasFloating(const IRPosition &IRP, Attributor &A)
- : AANoAliasImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // TODO: Implement this.
- return indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(noalias)
- }
-};
-
-/// NoAlias attribute for an argument.
-struct AANoAliasArgument final
- : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
- using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
- AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
-
- /// See AbstractAttribute::update(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // We have to make sure no-alias on the argument does not break
- // synchronization when this is a callback argument, see also [1] below.
- // If synchronization cannot be affected, we delegate to the base updateImpl
- // function, otherwise we give up for now.
-
- // If the function is no-sync, no-alias cannot break synchronization.
- bool IsKnownNoSycn;
- if (AA::hasAssumedIRAttr<Attribute::NoSync>(
- A, this, IRPosition::function_scope(getIRPosition()),
- DepClassTy::OPTIONAL, IsKnownNoSycn))
- return Base::updateImpl(A);
-
- // If the argument is read-only, no-alias cannot break synchronization.
- bool IsKnown;
- if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
- return Base::updateImpl(A);
-
- // If the argument is never passed through callbacks, no-alias cannot break
- // synchronization.
- bool UsedAssumedInformation = false;
- if (A.checkForAllCallSites(
- [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, *this,
- true, UsedAssumedInformation))
- return Base::updateImpl(A);
-
- // TODO: add no-alias but make sure it doesn't break synchronization by
- // introducing fake uses. See:
- // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
- // International Workshop on OpenMP 2018,
- // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
-
- return indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
-};
-
-struct AANoAliasCallSiteArgument final : AANoAliasImpl {
- AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AANoAliasImpl(IRP, A) {}
-
- /// Determine if the underlying value may alias with the call site argument
- /// \p OtherArgNo of \p ICS (= the underlying call site).
- bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
- const AAMemoryBehavior &MemBehaviorAA,
- const CallBase &CB, unsigned OtherArgNo) {
- // We do not need to worry about aliasing with the underlying IRP.
- if (this->getCalleeArgNo() == (int)OtherArgNo)
- return false;
-
- // If it is not a pointer or pointer vector we do not alias.
- const Value *ArgOp = CB.getArgOperand(OtherArgNo);
- if (!ArgOp->getType()->isPtrOrPtrVectorTy())
- return false;
-
- auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
- *this, IRPosition::callsite_argument(CB, OtherArgNo), DepClassTy::NONE);
-
- // If the argument is readnone, there is no read-write aliasing.
- if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
- A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);
- return false;
- }
-
- // If the argument is readonly and the underlying value is readonly, there
- // is no read-write aliasing.
- bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
- if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
- IsReadOnly) {
- A.recordDependence(MemBehaviorAA, *this, DepClassTy::OPTIONAL);
- A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);
- return false;
- }
-
- // We have to utilize actual alias analysis queries so we need the object.
- if (!AAR)
- AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
- *getAnchorScope());
-
- // Try to rule it out at the call site.
- bool IsAliasing = !AAR || !AAR->isNoAlias(&getAssociatedValue(), ArgOp);
- LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
- "callsite arguments: "
- << getAssociatedValue() << " " << *ArgOp << " => "
- << (IsAliasing ? "" : "no-") << "alias \n");
-
- return IsAliasing;
- }
-
- bool isKnownNoAliasDueToNoAliasPreservation(
- Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
- // We can deduce "noalias" if the following conditions hold.
- // (i) Associated value is assumed to be noalias in the definition.
- // (ii) Associated value is assumed to be no-capture in all the uses
- // possibly executed before this callsite.
- // (iii) There is no other pointer argument which could alias with the
- // value.
-
- auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
- const auto *DerefAA = A.getAAFor<AADereferenceable>(
- *this, IRPosition::value(*O), DepClassTy::OPTIONAL);
- return DerefAA ? DerefAA->getAssumedDereferenceableBytes() : 0;
- };
-
- const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
- const Function *ScopeFn = VIRP.getAnchorScope();
- // Check whether the value is captured in the scope using AANoCapture.
- // Look at CFG and check only uses possibly executed before this
- // callsite.
- auto UsePred = [&](const Use &U, bool &Follow) -> bool {
- Instruction *UserI = cast<Instruction>(U.getUser());
-
- // If UserI is the curr instruction and there is a single potential use of
- // the value in UserI we allow the use.
- // TODO: We should inspect the operands and allow those that cannot alias
- // with the value.
- if (UserI == getCtxI() && UserI->getNumOperands() == 1)
- return true;
-
- if (ScopeFn) {
- if (auto *CB = dyn_cast<CallBase>(UserI)) {
- if (CB->isArgOperand(&U)) {
-
- unsigned ArgNo = CB->getArgOperandNo(&U);
-
- bool IsKnownNoCapture;
- if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, IRPosition::callsite_argument(*CB, ArgNo),
- DepClassTy::OPTIONAL, IsKnownNoCapture))
- return true;
- }
- }
-
- if (!AA::isPotentiallyReachable(
- A, *UserI, *getCtxI(), *this, /* ExclusionSet */ nullptr,
- [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
- return true;
- }
-
- // TODO: We should track the capturing uses in AANoCapture but the problem
- // is CGSCC runs. For those we would need to "allow" AANoCapture for
- // a value in the module slice.
- switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) {
- case UseCaptureKind::NO_CAPTURE:
- return true;
- case UseCaptureKind::MAY_CAPTURE:
- LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI
- << "\n");
- return false;
- case UseCaptureKind::PASSTHROUGH:
- Follow = true;
- return true;
- }
- llvm_unreachable("unknown UseCaptureKind");
- };
-
- bool IsKnownNoCapture;
- const AANoCapture *NoCaptureAA = nullptr;
- bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, VIRP, DepClassTy::NONE, IsKnownNoCapture, false, &NoCaptureAA);
- if (!IsAssumedNoCapture &&
- (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
- if (!A.checkForAllUses(UsePred, *this, getAssociatedValue())) {
- LLVM_DEBUG(
- dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
- << " cannot be noalias as it is potentially captured\n");
- return false;
- }
- }
- if (NoCaptureAA)
- A.recordDependence(*NoCaptureAA, *this, DepClassTy::OPTIONAL);
-
- // Check there is no other pointer argument which could alias with the
- // value passed at this call site.
- // TODO: AbstractCallSite
- const auto &CB = cast<CallBase>(getAnchorValue());
- for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
- if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
- return false;
-
- return true;
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // If the argument is readnone we are done as there are no accesses via the
- // argument.
- auto *MemBehaviorAA =
- A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
- if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
- A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);
- return ChangeStatus::UNCHANGED;
- }
-
- bool IsKnownNoAlias;
- const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
- if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
- A, this, VIRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
- LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
- << " is not no-alias at the definition\n");
- return indicatePessimisticFixpoint();
- }
-
- AAResults *AAR = nullptr;
- if (MemBehaviorAA &&
- isKnownNoAliasDueToNoAliasPreservation(A, AAR, *MemBehaviorAA)) {
- LLVM_DEBUG(
- dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
- return ChangeStatus::UNCHANGED;
- }
-
- return indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
-};
-
-/// NoAlias attribute for function return value.
-struct AANoAliasReturned final : AANoAliasImpl {
- AANoAliasReturned(const IRPosition &IRP, Attributor &A)
- : AANoAliasImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
-
- auto CheckReturnValue = [&](Value &RV) -> bool {
- if (Constant *C = dyn_cast<Constant>(&RV))
- if (C->isNullValue() || isa<UndefValue>(C))
- return true;
-
- /// For now, we can only deduce noalias if we have call sites.
- /// FIXME: add more support.
- if (!isa<CallBase>(&RV))
- return false;
-
- const IRPosition &RVPos = IRPosition::value(RV);
- bool IsKnownNoAlias;
- if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
- A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoAlias))
- return false;
-
- bool IsKnownNoCapture;
- const AANoCapture *NoCaptureAA = nullptr;
- bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
- &NoCaptureAA);
- return IsAssumedNoCapture ||
- (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
- };
-
- if (!A.checkForAllReturnedValues(CheckReturnValue, *this))
- return indicatePessimisticFixpoint();
-
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
-};
-
-/// NoAlias attribute deduction for a call site return value.
-struct AANoAliasCallSiteReturned final
- : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
- AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
-};
-} // namespace
-
-/// -------------------AAIsDead Function Attribute-----------------------
-
-namespace {
-struct AAIsDeadValueImpl : public AAIsDead {
- AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
-
- /// See AAIsDead::isAssumedDead().
- bool isAssumedDead() const override { return isAssumed(IS_DEAD); }
-
- /// See AAIsDead::isKnownDead().
- bool isKnownDead() const override { return isKnown(IS_DEAD); }
-
- /// See AAIsDead::isAssumedDead(BasicBlock *).
- bool isAssumedDead(const BasicBlock *BB) const override { return false; }
-
- /// See AAIsDead::isKnownDead(BasicBlock *).
- bool isKnownDead(const BasicBlock *BB) const override { return false; }
-
- /// See AAIsDead::isAssumedDead(Instruction *I).
- bool isAssumedDead(const Instruction *I) const override {
- return I == getCtxI() && isAssumedDead();
- }
-
- /// See AAIsDead::isKnownDead(Instruction *I).
- bool isKnownDead(const Instruction *I) const override {
- return isAssumedDead(I) && isKnownDead();
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return isAssumedDead() ? "assumed-dead" : "assumed-live";
- }
-
- /// Check if all uses are assumed dead.
- bool areAllUsesAssumedDead(Attributor &A, Value &V) {
- // Callers might not check the type, void has no uses.
- if (V.getType()->isVoidTy() || V.use_empty())
- return true;
-
- // If we replace a value with a constant there are no uses left afterwards.
- if (!isa<Constant>(V)) {
- if (auto *I = dyn_cast<Instruction>(&V))
- if (!A.isRunOn(*I->getFunction()))
- return false;
- bool UsedAssumedInformation = false;
- std::optional<Constant *> C =
- A.getAssumedConstant(V, *this, UsedAssumedInformation);
- if (!C || *C)
- return true;
- }
-
- auto UsePred = [&](const Use &U, bool &Follow) { return false; };
- // Explicitly set the dependence class to required because we want a long
- // chain of N dependent instructions to be considered live as soon as one is
- // without going through N update cycles. This is not required for
- // correctness.
- return A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ false,
- DepClassTy::REQUIRED,
- /* IgnoreDroppableUses */ false);
- }
-
- /// Determine if \p I is assumed to be side-effect free.
- bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
- if (!I || wouldInstructionBeTriviallyDead(I))
- return true;
-
- auto *CB = dyn_cast<CallBase>(I);
- if (!CB || isa<IntrinsicInst>(CB))
- return false;
-
- const IRPosition &CallIRP = IRPosition::callsite_function(*CB);
-
- bool IsKnownNoUnwind;
- if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
- A, this, CallIRP, DepClassTy::OPTIONAL, IsKnownNoUnwind))
- return false;
-
- bool IsKnown;
- return AA::isAssumedReadOnly(A, CallIRP, *this, IsKnown);
- }
-};
-
-struct AAIsDeadFloating : public AAIsDeadValueImpl {
- AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
- : AAIsDeadValueImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- AAIsDeadValueImpl::initialize(A);
-
- if (isa<UndefValue>(getAssociatedValue())) {
- indicatePessimisticFixpoint();
- return;
- }
-
- Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
- if (!isAssumedSideEffectFree(A, I)) {
- if (!isa_and_nonnull<StoreInst>(I) && !isa_and_nonnull<FenceInst>(I))
- indicatePessimisticFixpoint();
- else
- removeAssumedBits(HAS_NO_EFFECT);
- }
- }
-
- bool isDeadFence(Attributor &A, FenceInst &FI) {
- const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
- IRPosition::function(*FI.getFunction()), *this, DepClassTy::NONE);
- if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
- return false;
- A.recordDependence(*ExecDomainAA, *this, DepClassTy::OPTIONAL);
- return true;
- }
-
- bool isDeadStore(Attributor &A, StoreInst &SI,
- SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
- // Lang ref now states volatile store is not UB/dead, let's skip them.
- if (SI.isVolatile())
- return false;
-
- // If we are collecting assumes to be deleted we are in the manifest stage.
- // It's problematic to collect the potential copies again now so we use the
- // cached ones.
- bool UsedAssumedInformation = false;
- if (!AssumeOnlyInst) {
- PotentialCopies.clear();
- if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, *this,
- UsedAssumedInformation)) {
- LLVM_DEBUG(
- dbgs()
- << "[AAIsDead] Could not determine potential copies of store!\n");
- return false;
- }
- }
- LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
- << " potential copies.\n");
-
- InformationCache &InfoCache = A.getInfoCache();
- return llvm::all_of(PotentialCopies, [&](Value *V) {
- if (A.isAssumedDead(IRPosition::value(*V), this, nullptr,
- UsedAssumedInformation))
- return true;
- if (auto *LI = dyn_cast<LoadInst>(V)) {
- if (llvm::all_of(LI->uses(), [&](const Use &U) {
- auto &UserI = cast<Instruction>(*U.getUser());
- if (InfoCache.isOnlyUsedByAssume(UserI)) {
- if (AssumeOnlyInst)
- AssumeOnlyInst->insert(&UserI);
- return true;
- }
- return A.isAssumedDead(U, this, nullptr, UsedAssumedInformation);
- })) {
- return true;
- }
- }
- LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
- << " is assumed live!\n");
- return false;
- });
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
- if (isa_and_nonnull<StoreInst>(I))
- if (isValidState())
- return "assumed-dead-store";
- if (isa_and_nonnull<FenceInst>(I))
- if (isValidState())
- return "assumed-dead-fence";
- return AAIsDeadValueImpl::getAsStr(A);
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
- if (auto *SI = dyn_cast_or_null<StoreInst>(I)) {
- if (!isDeadStore(A, *SI))
- return indicatePessimisticFixpoint();
- } else if (auto *FI = dyn_cast_or_null<FenceInst>(I)) {
- if (!isDeadFence(A, *FI))
- return indicatePessimisticFixpoint();
- } else {
- if (!isAssumedSideEffectFree(A, I))
- return indicatePessimisticFixpoint();
- if (!areAllUsesAssumedDead(A, getAssociatedValue()))
- return indicatePessimisticFixpoint();
- }
- return ChangeStatus::UNCHANGED;
- }
-
- bool isRemovableStore() const override {
- return isAssumed(IS_REMOVABLE) && isa<StoreInst>(&getAssociatedValue());
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- Value &V = getAssociatedValue();
- if (auto *I = dyn_cast<Instruction>(&V)) {
- // If we get here we basically know the users are all dead. We check if
- // isAssumedSideEffectFree returns true here again because it might not be
- // the case and only the users are dead but the instruction (=call) is
- // still needed.
- if (auto *SI = dyn_cast<StoreInst>(I)) {
- SmallSetVector<Instruction *, 8> AssumeOnlyInst;
- bool IsDead = isDeadStore(A, *SI, &AssumeOnlyInst);
- (void)IsDead;
- assert(IsDead && "Store was assumed to be dead!");
- A.deleteAfterManifest(*I);
- for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
- Instruction *AOI = AssumeOnlyInst[i];
- for (auto *Usr : AOI->users())
- AssumeOnlyInst.insert(cast<Instruction>(Usr));
- A.deleteAfterManifest(*AOI);
- }
- return ChangeStatus::CHANGED;
- }
- if (auto *FI = dyn_cast<FenceInst>(I)) {
- assert(isDeadFence(A, *FI));
- A.deleteAfterManifest(*FI);
- return ChangeStatus::CHANGED;
- }
- if (isAssumedSideEffectFree(A, I) && !isa<InvokeInst>(I)) {
- A.deleteAfterManifest(*I);
- return ChangeStatus::CHANGED;
- }
- }
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(IsDead)
- }
-
-private:
- // The potential copies of a dead store, used for deletion during manifest.
- SmallSetVector<Value *, 4> PotentialCopies;
-};
-
-struct AAIsDeadArgument : public AAIsDeadFloating {
- AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
- : AAIsDeadFloating(IRP, A) {}
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- Argument &Arg = *getAssociatedArgument();
- if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
- if (A.registerFunctionSignatureRewrite(
- Arg, /* ReplacementTypes */ {},
- Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{},
- Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) {
- return ChangeStatus::CHANGED;
- }
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
-};
-
-struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
- AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAIsDeadValueImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- AAIsDeadValueImpl::initialize(A);
- if (isa<UndefValue>(getAssociatedValue()))
- indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // TODO: Once we have call site specific value information we can provide
- // call site specific liveness information and then it makes
- // sense to specialize attributes for call sites arguments instead of
- // redirecting requests to the callee argument.
- Argument *Arg = getAssociatedArgument();
- if (!Arg)
- return indicatePessimisticFixpoint();
- const IRPosition &ArgPos = IRPosition::argument(*Arg);
- auto *ArgAA = A.getAAFor<AAIsDead>(*this, ArgPos, DepClassTy::REQUIRED);
- if (!ArgAA)
- return indicatePessimisticFixpoint();
- return clampStateAndIndicateChange(getState(), ArgAA->getState());
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- CallBase &CB = cast<CallBase>(getAnchorValue());
- Use &U = CB.getArgOperandUse(getCallSiteArgNo());
- assert(!isa<UndefValue>(U.get()) &&
- "Expected undef values to be filtered out!");
- UndefValue &UV = *UndefValue::get(U->getType());
- if (A.changeUseAfterManifest(U, UV))
- return ChangeStatus::CHANGED;
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
-};
-
-struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
- AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AAIsDeadFloating(IRP, A) {}
-
- /// See AAIsDead::isAssumedDead().
- bool isAssumedDead() const override {
- return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
- }
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- AAIsDeadFloating::initialize(A);
- if (isa<UndefValue>(getAssociatedValue())) {
- indicatePessimisticFixpoint();
- return;
- }
-
- // We track this separately as a secondary state.
- IsAssumedSideEffectFree = isAssumedSideEffectFree(A, getCtxI());
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, getCtxI())) {
- IsAssumedSideEffectFree = false;
- Changed = ChangeStatus::CHANGED;
- }
- if (!areAllUsesAssumedDead(A, getAssociatedValue()))
- return indicatePessimisticFixpoint();
- return Changed;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- if (IsAssumedSideEffectFree)
- STATS_DECLTRACK_CSRET_ATTR(IsDead)
- else
- STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return isAssumedDead()
- ? "assumed-dead"
- : (getAssumed() ? "assumed-dead-users" : "assumed-live");
- }
-
-private:
- bool IsAssumedSideEffectFree = true;
-};
-
-struct AAIsDeadReturned : public AAIsDeadValueImpl {
- AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
- : AAIsDeadValueImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
-
- bool UsedAssumedInformation = false;
- A.checkForAllInstructions([](Instruction &) { return true; }, *this,
- {Instruction::Ret}, UsedAssumedInformation);
-
- auto PredForCallSite = [&](AbstractCallSite ACS) {
- if (ACS.isCallbackCall() || !ACS.getInstruction())
- return false;
- return areAllUsesAssumedDead(A, *ACS.getInstruction());
- };
-
- if (!A.checkForAllCallSites(PredForCallSite, *this, true,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- // TODO: Rewrite the signature to return void?
- bool AnyChange = false;
- UndefValue &UV = *UndefValue::get(getAssociatedFunction()->getReturnType());
- auto RetInstPred = [&](Instruction &I) {
- ReturnInst &RI = cast<ReturnInst>(I);
- if (!isa<UndefValue>(RI.getReturnValue()))
- AnyChange |= A.changeUseAfterManifest(RI.getOperandUse(0), UV);
- return true;
- };
- bool UsedAssumedInformation = false;
- A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
- UsedAssumedInformation);
- return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
-};
-
-struct AAIsDeadFunction : public AAIsDead {
- AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- Function *F = getAnchorScope();
- assert(F && "Did expect an anchor function");
- if (!isAssumedDeadInternalFunction(A)) {
- ToBeExploredFrom.insert(&F->getEntryBlock().front());
- assumeLive(A, F->getEntryBlock());
- }
- }
-
- bool isAssumedDeadInternalFunction(Attributor &A) {
- if (!getAnchorScope()->hasLocalLinkage())
- return false;
- bool UsedAssumedInformation = false;
- return A.checkForAllCallSites([](AbstractCallSite) { return false; }, *this,
- true, UsedAssumedInformation);
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return "Live[#BB " + std::to_string(AssumedLiveBlocks.size()) + "/" +
- std::to_string(getAnchorScope()->size()) + "][#TBEP " +
- std::to_string(ToBeExploredFrom.size()) + "][#KDE " +
- std::to_string(KnownDeadEnds.size()) + "]";
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- assert(getState().isValidState() &&
- "Attempted to manifest an invalid state!");
-
- ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
- Function &F = *getAnchorScope();
-
- if (AssumedLiveBlocks.empty()) {
- A.deleteAfterManifest(F);
- return ChangeStatus::CHANGED;
- }
-
- // Flag to determine if we can change an invoke to a call assuming the
- // callee is nounwind. This is not possible if the personality of the
- // function allows to catch asynchronous exceptions.
- bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
-
- KnownDeadEnds.set_union(ToBeExploredFrom);
- for (const Instruction *DeadEndI : KnownDeadEnds) {
- auto *CB = dyn_cast<CallBase>(DeadEndI);
- if (!CB)
- continue;
- bool IsKnownNoReturn;
- bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
- A, this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL,
- IsKnownNoReturn);
- if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(CB)))
- continue;
-
- if (auto *II = dyn_cast<InvokeInst>(DeadEndI))
- A.registerInvokeWithDeadSuccessor(const_cast<InvokeInst &>(*II));
- else
- A.changeToUnreachableAfterManifest(
- const_cast<Instruction *>(DeadEndI->getNextNode()));
- HasChanged = ChangeStatus::CHANGED;
- }
-
- STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
- for (BasicBlock &BB : F)
- if (!AssumedLiveBlocks.count(&BB)) {
- A.deleteAfterManifest(BB);
- ++BUILD_STAT_NAME(AAIsDead, BasicBlock);
- HasChanged = ChangeStatus::CHANGED;
- }
-
- return HasChanged;
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override;
-
- bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
- assert(From->getParent() == getAnchorScope() &&
- To->getParent() == getAnchorScope() &&
- "Used AAIsDead of the wrong function");
- return isValidState() && !AssumedLiveEdges.count(std::make_pair(From, To));
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {}
-
- /// Returns true if the function is assumed dead.
- bool isAssumedDead() const override { return false; }
-
- /// See AAIsDead::isKnownDead().
- bool isKnownDead() const override { return false; }
-
- /// See AAIsDead::isAssumedDead(BasicBlock *).
- bool isAssumedDead(const BasicBlock *BB) const override {
- assert(BB->getParent() == getAnchorScope() &&
- "BB must be in the same anchor scope function.");
-
- if (!getAssumed())
- return false;
- return !AssumedLiveBlocks.count(BB);
- }
-
- /// See AAIsDead::isKnownDead(BasicBlock *).
- bool isKnownDead(const BasicBlock *BB) const override {
- return getKnown() && isAssumedDead(BB);
- }
-
- /// See AAIsDead::isAssumed(Instruction *I).
- bool isAssumedDead(const Instruction *I) const override {
- assert(I->getParent()->getParent() == getAnchorScope() &&
- "Instruction must be in the same anchor scope function.");
-
- if (!getAssumed())
- return false;
-
- // If it is not in AssumedLiveBlocks then it for sure dead.
- // Otherwise, it can still be after noreturn call in a live block.
- if (!AssumedLiveBlocks.count(I->getParent()))
- return true;
-
- // If it is not after a liveness barrier it is live.
- const Instruction *PrevI = I->getPrevNode();
- while (PrevI) {
- if (KnownDeadEnds.count(PrevI) || ToBeExploredFrom.count(PrevI))
- return true;
- PrevI = PrevI->getPrevNode();
- }
- return false;
- }
-
- /// See AAIsDead::isKnownDead(Instruction *I).
- bool isKnownDead(const Instruction *I) const override {
- return getKnown() && isAssumedDead(I);
- }
-
- /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
- /// that internal function called from \p BB should now be looked at.
- bool assumeLive(Attributor &A, const BasicBlock &BB) {
- if (!AssumedLiveBlocks.insert(&BB).second)
- return false;
-
- // We assume that all of BB is (probably) live now and if there are calls to
- // internal functions we will assume that those are now live as well. This
- // is a performance optimization for blocks with calls to a lot of internal
- // functions. It can however cause dead functions to be treated as live.
- for (const Instruction &I : BB)
- if (const auto *CB = dyn_cast<CallBase>(&I))
- if (auto *F = dyn_cast_if_present<Function>(CB->getCalledOperand()))
- if (F->hasLocalLinkage())
- A.markLiveInternalFunction(*F);
- return true;
- }
-
- /// Collection of instructions that need to be explored again, e.g., we
- /// did assume they do not transfer control to (one of their) successors.
- SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
-
- /// Collection of instructions that are known to not transfer control.
- SmallSetVector<const Instruction *, 8> KnownDeadEnds;
-
- /// Collection of all assumed live edges
- DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
-
- /// Collection of all assumed live BasicBlocks.
- DenseSet<const BasicBlock *> AssumedLiveBlocks;
-};
-
-static bool
-identifyAliveSuccessors(Attributor &A, const CallBase &CB,
- AbstractAttribute &AA,
- SmallVectorImpl<const Instruction *> &AliveSuccessors) {
- const IRPosition &IPos = IRPosition::callsite_function(CB);
-
- bool IsKnownNoReturn;
- if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
- A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoReturn))
- return !IsKnownNoReturn;
- if (CB.isTerminator())
- AliveSuccessors.push_back(&CB.getSuccessor(0)->front());
- else
- AliveSuccessors.push_back(CB.getNextNode());
- return false;
-}
-
-static bool
-identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
- AbstractAttribute &AA,
- SmallVectorImpl<const Instruction *> &AliveSuccessors) {
- bool UsedAssumedInformation =
- identifyAliveSuccessors(A, cast<CallBase>(II), AA, AliveSuccessors);
-
- // First, determine if we can change an invoke to a call assuming the
- // callee is nounwind. This is not possible if the personality of the
- // function allows to catch asynchronous exceptions.
- if (AAIsDeadFunction::mayCatchAsynchronousExceptions(*II.getFunction())) {
- AliveSuccessors.push_back(&II.getUnwindDest()->front());
- } else {
- const IRPosition &IPos = IRPosition::callsite_function(II);
-
- bool IsKnownNoUnwind;
- if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
- A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
- UsedAssumedInformation |= !IsKnownNoUnwind;
- } else {
- AliveSuccessors.push_back(&II.getUnwindDest()->front());
- }
- }
- return UsedAssumedInformation;
-}
-
-static bool
-identifyAliveSuccessors(Attributor &A, const BranchInst &BI,
- AbstractAttribute &AA,
- SmallVectorImpl<const Instruction *> &AliveSuccessors) {
- bool UsedAssumedInformation = false;
- if (BI.getNumSuccessors() == 1) {
- AliveSuccessors.push_back(&BI.getSuccessor(0)->front());
- } else {
- std::optional<Constant *> C =
- A.getAssumedConstant(*BI.getCondition(), AA, UsedAssumedInformation);
- if (!C || isa_and_nonnull<UndefValue>(*C)) {
- // No value yet, assume both edges are dead.
- } else if (isa_and_nonnull<ConstantInt>(*C)) {
- const BasicBlock *SuccBB =
- BI.getSuccessor(1 - cast<ConstantInt>(*C)->getValue().getZExtValue());
- AliveSuccessors.push_back(&SuccBB->front());
- } else {
- AliveSuccessors.push_back(&BI.getSuccessor(0)->front());
- AliveSuccessors.push_back(&BI.getSuccessor(1)->front());
- UsedAssumedInformation = false;
- }
- }
- return UsedAssumedInformation;
-}
-
-static bool
-identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
- AbstractAttribute &AA,
- SmallVectorImpl<const Instruction *> &AliveSuccessors) {
- bool UsedAssumedInformation = false;
- SmallVector<AA::ValueAndContext> Values;
- if (!A.getAssumedSimplifiedValues(IRPosition::value(*SI.getCondition()), &AA,
- Values, AA::AnyScope,
- UsedAssumedInformation)) {
- // Something went wrong, assume all successors are live.
- for (const BasicBlock *SuccBB : successors(SI.getParent()))
- AliveSuccessors.push_back(&SuccBB->front());
- return false;
- }
-
- if (Values.empty() ||
- (Values.size() == 1 &&
- isa_and_nonnull<UndefValue>(Values.front().getValue()))) {
- // No valid value yet, assume all edges are dead.
- return UsedAssumedInformation;
- }
-
- Type &Ty = *SI.getCondition()->getType();
- SmallPtrSet<ConstantInt *, 8> Constants;
- auto CheckForConstantInt = [&](Value *V) {
- if (auto *CI = dyn_cast_if_present<ConstantInt>(AA::getWithType(*V, Ty))) {
- Constants.insert(CI);
- return true;
- }
- return false;
- };
-
- if (!all_of(Values, [&](AA::ValueAndContext &VAC) {
- return CheckForConstantInt(VAC.getValue());
- })) {
- for (const BasicBlock *SuccBB : successors(SI.getParent()))
- AliveSuccessors.push_back(&SuccBB->front());
- return UsedAssumedInformation;
- }
-
- unsigned MatchedCases = 0;
- for (const auto &CaseIt : SI.cases()) {
- if (Constants.count(CaseIt.getCaseValue())) {
- ++MatchedCases;
- AliveSuccessors.push_back(&CaseIt.getCaseSuccessor()->front());
- }
- }
-
- // If all potential values have been matched, we will not visit the default
- // case.
- if (MatchedCases < Constants.size())
- AliveSuccessors.push_back(&SI.getDefaultDest()->front());
- return UsedAssumedInformation;
-}
-
-ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
- ChangeStatus Change = ChangeStatus::UNCHANGED;
-
- if (AssumedLiveBlocks.empty()) {
- if (isAssumedDeadInternalFunction(A))
- return ChangeStatus::UNCHANGED;
-
- Function *F = getAnchorScope();
- ToBeExploredFrom.insert(&F->getEntryBlock().front());
- assumeLive(A, F->getEntryBlock());
- Change = ChangeStatus::CHANGED;
- }
-
- LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
- << getAnchorScope()->size() << "] BBs and "
- << ToBeExploredFrom.size() << " exploration points and "
- << KnownDeadEnds.size() << " known dead ends\n");
-
- // Copy and clear the list of instructions we need to explore from. It is
- // refilled with instructions the next update has to look at.
- SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
- ToBeExploredFrom.end());
- decltype(ToBeExploredFrom) NewToBeExploredFrom;
-
- SmallVector<const Instruction *, 8> AliveSuccessors;
- while (!Worklist.empty()) {
- const Instruction *I = Worklist.pop_back_val();
- LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
-
- // Fast forward for uninteresting instructions. We could look for UB here
- // though.
- while (!I->isTerminator() && !isa<CallBase>(I))
- I = I->getNextNode();
-
- AliveSuccessors.clear();
-
- bool UsedAssumedInformation = false;
- switch (I->getOpcode()) {
- // TODO: look for (assumed) UB to backwards propagate "deadness".
- default:
- assert(I->isTerminator() &&
- "Expected non-terminators to be handled already!");
- for (const BasicBlock *SuccBB : successors(I->getParent()))
- AliveSuccessors.push_back(&SuccBB->front());
- break;
- case Instruction::Call:
- UsedAssumedInformation = identifyAliveSuccessors(A, cast<CallInst>(*I),
- *this, AliveSuccessors);
- break;
- case Instruction::Invoke:
- UsedAssumedInformation = identifyAliveSuccessors(A, cast<InvokeInst>(*I),
- *this, AliveSuccessors);
- break;
- case Instruction::Br:
- UsedAssumedInformation = identifyAliveSuccessors(A, cast<BranchInst>(*I),
- *this, AliveSuccessors);
- break;
- case Instruction::Switch:
- UsedAssumedInformation = identifyAliveSuccessors(A, cast<SwitchInst>(*I),
- *this, AliveSuccessors);
- break;
- }
-
- if (UsedAssumedInformation) {
- NewToBeExploredFrom.insert(I);
- } else if (AliveSuccessors.empty() ||
- (I->isTerminator() &&
- AliveSuccessors.size() < I->getNumSuccessors())) {
- if (KnownDeadEnds.insert(I))
- Change = ChangeStatus::CHANGED;
- }
-
- LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
- << AliveSuccessors.size() << " UsedAssumedInformation: "
- << UsedAssumedInformation << "\n");
-
- for (const Instruction *AliveSuccessor : AliveSuccessors) {
- if (!I->isTerminator()) {
- assert(AliveSuccessors.size() == 1 &&
- "Non-terminator expected to have a single successor!");
- Worklist.push_back(AliveSuccessor);
- } else {
- // record the assumed live edge
- auto Edge = std::make_pair(I->getParent(), AliveSuccessor->getParent());
- if (AssumedLiveEdges.insert(Edge).second)
- Change = ChangeStatus::CHANGED;
- if (assumeLive(A, *AliveSuccessor->getParent()))
- Worklist.push_back(AliveSuccessor);
- }
- }
- }
-
- // Check if the content of ToBeExploredFrom changed, ignore the order.
- if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
- llvm::any_of(NewToBeExploredFrom, [&](const Instruction *I) {
- return !ToBeExploredFrom.count(I);
- })) {
- Change = ChangeStatus::CHANGED;
- ToBeExploredFrom = std::move(NewToBeExploredFrom);
- }
-
- // If we know everything is live there is no need to query for liveness.
- // Instead, indicating a pessimistic fixpoint will cause the state to be
- // "invalid" and all queries to be answered conservatively without lookups.
- // To be in this state we have to (1) finished the exploration and (3) not
- // discovered any non-trivial dead end and (2) not ruled unreachable code
- // dead.
- if (ToBeExploredFrom.empty() &&
- getAnchorScope()->size() == AssumedLiveBlocks.size() &&
- llvm::all_of(KnownDeadEnds, [](const Instruction *DeadEndI) {
- return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
- }))
- return indicatePessimisticFixpoint();
- return Change;
-}
-
-/// Liveness information for a call sites.
-struct AAIsDeadCallSite final : AAIsDeadFunction {
- AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
- : AAIsDeadFunction(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- // TODO: Once we have call site specific value information we can provide
- // call site specific liveness information and then it makes
- // sense to specialize attributes for call sites instead of
- // redirecting requests to the callee.
- llvm_unreachable("Abstract attributes for liveness are not "
- "supported for call sites yet!");
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- return indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {}
-};
-} // namespace
-
-/// -------------------- Dereferenceable Argument Attribute --------------------
-
-namespace {
-struct AADereferenceableImpl : AADereferenceable {
- AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
- : AADereferenceable(IRP, A) {}
- using StateType = DerefState;
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- Value &V = *getAssociatedValue().stripPointerCasts();
- SmallVector<Attribute, 4> Attrs;
- A.getAttrs(getIRPosition(),
- {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
- Attrs, /* IgnoreSubsumingPositions */ false);
- for (const Attribute &Attr : Attrs)
- takeKnownDerefBytesMaximum(Attr.getValueAsInt());
-
- // Ensure we initialize the non-null AA (if necessary).
- bool IsKnownNonNull;
- AA::hasAssumedIRAttr<Attribute::NonNull>(
- A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNonNull);
-
- bool CanBeNull, CanBeFreed;
- takeKnownDerefBytesMaximum(V.getPointerDereferenceableBytes(
- A.getDataLayout(), CanBeNull, CanBeFreed));
-
- if (Instruction *CtxI = getCtxI())
- followUsesInMBEC(*this, A, getState(), *CtxI);
- }
-
- /// See AbstractAttribute::getState()
- /// {
- StateType &getState() override { return *this; }
- const StateType &getState() const override { return *this; }
- /// }
-
- /// Helper function for collecting accessed bytes in must-be-executed-context
- void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
- DerefState &State) {
- const Value *UseV = U->get();
- if (!UseV->getType()->isPointerTy())
- return;
-
- std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);
- if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
- return;
-
- int64_t Offset;
- const Value *Base = GetPointerBaseWithConstantOffset(
- Loc->Ptr, Offset, A.getDataLayout(), /*AllowNonInbounds*/ true);
- if (Base && Base == &getAssociatedValue())
- State.addAccessedBytes(Offset, Loc->Size.getValue());
- }
-
- /// See followUsesInMBEC
- bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
- AADereferenceable::StateType &State) {
- bool IsNonNull = false;
- bool TrackUse = false;
- int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
- A, *this, getAssociatedValue(), U, I, IsNonNull, TrackUse);
- LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
- << " for instruction " << *I << "\n");
-
- addAccessedBytesForUse(A, U, I, State);
- State.takeKnownDerefBytesMaximum(DerefBytes);
- return TrackUse;
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- ChangeStatus Change = AADereferenceable::manifest(A);
- bool IsKnownNonNull;
- bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
- A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
- if (IsAssumedNonNull &&
- A.hasAttr(getIRPosition(), Attribute::DereferenceableOrNull)) {
- A.removeAttrs(getIRPosition(), {Attribute::DereferenceableOrNull});
- return ChangeStatus::CHANGED;
- }
- return Change;
- }
-
- void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
- SmallVectorImpl<Attribute> &Attrs) const override {
- // TODO: Add *_globally support
- bool IsKnownNonNull;
- bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
- A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
- if (IsAssumedNonNull)
- Attrs.emplace_back(Attribute::getWithDereferenceableBytes(
- Ctx, getAssumedDereferenceableBytes()));
- else
- Attrs.emplace_back(Attribute::getWithDereferenceableOrNullBytes(
- Ctx, getAssumedDereferenceableBytes()));
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- if (!getAssumedDereferenceableBytes())
- return "unknown-dereferenceable";
- bool IsKnownNonNull;
- bool IsAssumedNonNull = false;
- if (A)
- IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
- *A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
- return std::string("dereferenceable") +
- (IsAssumedNonNull ? "" : "_or_null") +
- (isAssumedGlobal() ? "_globally" : "") + "<" +
- std::to_string(getKnownDereferenceableBytes()) + "-" +
- std::to_string(getAssumedDereferenceableBytes()) + ">" +
- (!A ? " [non-null is unknown]" : "");
- }
-};
-
-/// Dereferenceable attribute for a floating value.
-struct AADereferenceableFloating : AADereferenceableImpl {
- AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
- : AADereferenceableImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- bool Stripped;
- bool UsedAssumedInformation = false;
- SmallVector<AA::ValueAndContext> Values;
- if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
- AA::AnyScope, UsedAssumedInformation)) {
- Values.push_back({getAssociatedValue(), getCtxI()});
- Stripped = false;
- } else {
- Stripped = Values.size() != 1 ||
- Values.front().getValue() != &getAssociatedValue();
- }
-
- const DataLayout &DL = A.getDataLayout();
- DerefState T;
-
- auto VisitValueCB = [&](const Value &V) -> bool {
- unsigned IdxWidth =
- DL.getIndexSizeInBits(V.getType()->getPointerAddressSpace());
- APInt Offset(IdxWidth, 0);
- const Value *Base = stripAndAccumulateOffsets(
- A, *this, &V, DL, Offset, /* GetMinOffset */ false,
- /* AllowNonInbounds */ true);
-
- const auto *AA = A.getAAFor<AADereferenceable>(
- *this, IRPosition::value(*Base), DepClassTy::REQUIRED);
- int64_t DerefBytes = 0;
- if (!AA || (!Stripped && this == AA)) {
- // Use IR information if we did not strip anything.
- // TODO: track globally.
- bool CanBeNull, CanBeFreed;
- DerefBytes =
- Base->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
- T.GlobalState.indicatePessimisticFixpoint();
- } else {
- const DerefState &DS = AA->getState();
- DerefBytes = DS.DerefBytesState.getAssumed();
- T.GlobalState &= DS.GlobalState;
- }
-
- // For now we do not try to "increase" dereferenceability due to negative
- // indices as we first have to come up with code to deal with loops and
- // for overflows of the dereferenceable bytes.
- int64_t OffsetSExt = Offset.getSExtValue();
- if (OffsetSExt < 0)
- OffsetSExt = 0;
-
- T.takeAssumedDerefBytesMinimum(
- std::max(int64_t(0), DerefBytes - OffsetSExt));
-
- if (this == AA) {
- if (!Stripped) {
- // If nothing was stripped IR information is all we got.
- T.takeKnownDerefBytesMaximum(
- std::max(int64_t(0), DerefBytes - OffsetSExt));
- T.indicatePessimisticFixpoint();
- } else if (OffsetSExt > 0) {
- // If something was stripped but there is circular reasoning we look
- // for the offset. If it is positive we basically decrease the
- // dereferenceable bytes in a circular loop now, which will simply
- // drive them down to the known value in a very slow way which we
- // can accelerate.
- T.indicatePessimisticFixpoint();
- }
- }
-
- return T.isValidState();
- };
-
- for (const auto &VAC : Values)
- if (!VisitValueCB(*VAC.getValue()))
- return indicatePessimisticFixpoint();
-
- return clampStateAndIndicateChange(getState(), T);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
- }
-};
-
-/// Dereferenceable attribute for a return value.
-struct AADereferenceableReturned final
- : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
- using Base =
- AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
- AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
- }
-};
-
-/// Dereferenceable attribute for an argument
-struct AADereferenceableArgument final
- : AAArgumentFromCallSiteArguments<AADereferenceable,
- AADereferenceableImpl> {
- using Base =
- AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
- AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_ARG_ATTR(dereferenceable)
- }
-};
-
-/// Dereferenceable attribute for a call site argument.
-struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
- AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AADereferenceableFloating(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
- }
-};
-
-/// Dereferenceable attribute deduction for a call site return value.
-struct AADereferenceableCallSiteReturned final
- : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
- using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
- AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CS_ATTR(dereferenceable);
- }
-};
-} // namespace
-
-// ------------------------ Align Argument Attribute ------------------------
-
-namespace {
-static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
- Value &AssociatedValue, const Use *U,
- const Instruction *I, bool &TrackUse) {
- // We need to follow common pointer manipulation uses to the accesses they
- // feed into.
- if (isa<CastInst>(I)) {
- // Follow all but ptr2int casts.
- TrackUse = !isa<PtrToIntInst>(I);
- return 0;
- }
- if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
- if (GEP->hasAllConstantIndices())
- TrackUse = true;
- return 0;
- }
-
- MaybeAlign MA;
- if (const auto *CB = dyn_cast<CallBase>(I)) {
- if (CB->isBundleOperand(U) || CB->isCallee(U))
- return 0;
-
- unsigned ArgNo = CB->getArgOperandNo(U);
- IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
- // As long as we only use known information there is no need to track
- // dependences here.
- auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClassTy::NONE);
- if (AlignAA)
- MA = MaybeAlign(AlignAA->getKnownAlign());
- }
-
- const DataLayout &DL = A.getDataLayout();
- const Value *UseV = U->get();
- if (auto *SI = dyn_cast<StoreInst>(I)) {
- if (SI->getPointerOperand() == UseV)
- MA = SI->getAlign();
- } else if (auto *LI = dyn_cast<LoadInst>(I)) {
- if (LI->getPointerOperand() == UseV)
- MA = LI->getAlign();
- } else if (auto *AI = dyn_cast<AtomicRMWInst>(I)) {
- if (AI->getPointerOperand() == UseV)
- MA = AI->getAlign();
- } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
- if (AI->getPointerOperand() == UseV)
- MA = AI->getAlign();
- }
-
- if (!MA || *MA <= QueryingAA.getKnownAlign())
- return 0;
-
- unsigned Alignment = MA->value();
- int64_t Offset;
-
- if (const Value *Base = GetPointerBaseWithConstantOffset(UseV, Offset, DL)) {
- if (Base == &AssociatedValue) {
- // BasePointerAddr + Offset = Alignment * Q for some integer Q.
- // So we can say that the maximum power of two which is a divisor of
- // gcd(Offset, Alignment) is an alignment.
-
- uint32_t gcd = std::gcd(uint32_t(abs((int32_t)Offset)), Alignment);
- Alignment = llvm::bit_floor(gcd);
- }
- }
-
- return Alignment;
-}
-
-struct AAAlignImpl : AAAlign {
- AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- SmallVector<Attribute, 4> Attrs;
- A.getAttrs(getIRPosition(), {Attribute::Alignment}, Attrs);
- for (const Attribute &Attr : Attrs)
- takeKnownMaximum(Attr.getValueAsInt());
-
- Value &V = *getAssociatedValue().stripPointerCasts();
- takeKnownMaximum(V.getPointerAlignment(A.getDataLayout()).value());
-
- if (Instruction *CtxI = getCtxI())
- followUsesInMBEC(*this, A, getState(), *CtxI);
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- ChangeStatus LoadStoreChanged = ChangeStatus::UNCHANGED;
-
- // Check for users that allow alignment annotations.
- Value &AssociatedValue = getAssociatedValue();
- for (const Use &U : AssociatedValue.uses()) {
- if (auto *SI = dyn_cast<StoreInst>(U.getUser())) {
- if (SI->getPointerOperand() == &AssociatedValue)
- if (SI->getAlign() < getAssumedAlign()) {
- STATS_DECLTRACK(AAAlign, Store,
- "Number of times alignment added to a store");
- SI->setAlignment(getAssumedAlign());
- LoadStoreChanged = ChangeStatus::CHANGED;
- }
- } else if (auto *LI = dyn_cast<LoadInst>(U.getUser())) {
- if (LI->getPointerOperand() == &AssociatedValue)
- if (LI->getAlign() < getAssumedAlign()) {
- LI->setAlignment(getAssumedAlign());
- STATS_DECLTRACK(AAAlign, Load,
- "Number of times alignment added to a load");
- LoadStoreChanged = ChangeStatus::CHANGED;
- }
- }
- }
-
- ChangeStatus Changed = AAAlign::manifest(A);
-
- Align InheritAlign =
- getAssociatedValue().getPointerAlignment(A.getDataLayout());
- if (InheritAlign >= getAssumedAlign())
- return LoadStoreChanged;
- return Changed | LoadStoreChanged;
- }
-
- // TODO: Provide a helper to determine the implied ABI alignment and check in
- // the existing manifest method and a new one for AAAlignImpl that value
- // to avoid making the alignment explicit if it did not improve.
-
- /// See AbstractAttribute::getDeducedAttributes
- void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
- SmallVectorImpl<Attribute> &Attrs) const override {
- if (getAssumedAlign() > 1)
- Attrs.emplace_back(
- Attribute::getWithAlignment(Ctx, Align(getAssumedAlign())));
- }
-
- /// See followUsesInMBEC
- bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
- AAAlign::StateType &State) {
- bool TrackUse = false;
-
- unsigned int KnownAlign =
- getKnownAlignForUse(A, *this, getAssociatedValue(), U, I, TrackUse);
- State.takeKnownMaximum(KnownAlign);
-
- return TrackUse;
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return "align<" + std::to_string(getKnownAlign().value()) + "-" +
- std::to_string(getAssumedAlign().value()) + ">";
- }
-};
-
-/// Align attribute for a floating value.
-struct AAAlignFloating : AAAlignImpl {
- AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- const DataLayout &DL = A.getDataLayout();
-
- bool Stripped;
- bool UsedAssumedInformation = false;
- SmallVector<AA::ValueAndContext> Values;
- if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
- AA::AnyScope, UsedAssumedInformation)) {
- Values.push_back({getAssociatedValue(), getCtxI()});
- Stripped = false;
- } else {
- Stripped = Values.size() != 1 ||
- Values.front().getValue() != &getAssociatedValue();
- }
-
- StateType T;
- auto VisitValueCB = [&](Value &V) -> bool {
- if (isa<UndefValue>(V) || isa<ConstantPointerNull>(V))
- return true;
- const auto *AA = A.getAAFor<AAAlign>(*this, IRPosition::value(V),
- DepClassTy::REQUIRED);
- if (!AA || (!Stripped && this == AA)) {
- int64_t Offset;
- unsigned Alignment = 1;
- if (const Value *Base =
- GetPointerBaseWithConstantOffset(&V, Offset, DL)) {
- // TODO: Use AAAlign for the base too.
- Align PA = Base->getPointerAlignment(DL);
- // BasePointerAddr + Offset = Alignment * Q for some integer Q.
- // So we can say that the maximum power of two which is a divisor of
- // gcd(Offset, Alignment) is an alignment.
-
- uint32_t gcd =
- std::gcd(uint32_t(abs((int32_t)Offset)), uint32_t(PA.value()));
- Alignment = llvm::bit_floor(gcd);
- } else {
- Alignment = V.getPointerAlignment(DL).value();
- }
- // Use only IR information if we did not strip anything.
- T.takeKnownMaximum(Alignment);
- T.indicatePessimisticFixpoint();
- } else {
- // Use abstract attribute information.
- const AAAlign::StateType &DS = AA->getState();
- T ^= DS;
- }
- return T.isValidState();
- };
-
- for (const auto &VAC : Values) {
- if (!VisitValueCB(*VAC.getValue()))
- return indicatePessimisticFixpoint();
- }
-
- // TODO: If we know we visited all incoming values, thus no are assumed
- // dead, we can take the known information from the state T.
- return clampStateAndIndicateChange(getState(), T);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
-};
-
-/// Align attribute for function return value.
-struct AAAlignReturned final
- : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
- using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
- AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
-};
-
-/// Align attribute for function argument.
-struct AAAlignArgument final
- : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
- using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
- AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- // If the associated argument is involved in a must-tail call we give up
- // because we would need to keep the argument alignments of caller and
- // callee in-sync. Just does not seem worth the trouble right now.
- if (A.getInfoCache().isInvolvedInMustTailCall(*getAssociatedArgument()))
- return ChangeStatus::UNCHANGED;
- return Base::manifest(A);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
-};
-
-struct AAAlignCallSiteArgument final : AAAlignFloating {
- AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAAlignFloating(IRP, A) {}
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- // If the associated argument is involved in a must-tail call we give up
- // because we would need to keep the argument alignments of caller and
- // callee in-sync. Just does not seem worth the trouble right now.
- if (Argument *Arg = getAssociatedArgument())
- if (A.getInfoCache().isInvolvedInMustTailCall(*Arg))
- return ChangeStatus::UNCHANGED;
- ChangeStatus Changed = AAAlignImpl::manifest(A);
- Align InheritAlign =
- getAssociatedValue().getPointerAlignment(A.getDataLayout());
- if (InheritAlign >= getAssumedAlign())
- Changed = ChangeStatus::UNCHANGED;
- return Changed;
- }
-
- /// See AbstractAttribute::updateImpl(Attributor &A).
- ChangeStatus updateImpl(Attributor &A) override {
- ChangeStatus Changed = AAAlignFloating::updateImpl(A);
- if (Argument *Arg = getAssociatedArgument()) {
- // We only take known information from the argument
- // so we do not need to track a dependence.
- const auto *ArgAlignAA = A.getAAFor<AAAlign>(
- *this, IRPosition::argument(*Arg), DepClassTy::NONE);
- if (ArgAlignAA)
- takeKnownMaximum(ArgAlignAA->getKnownAlign().value());
- }
- return Changed;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
-};
-
-/// Align attribute deduction for a call site return value.
-struct AAAlignCallSiteReturned final
- : AACalleeToCallSite<AAAlign, AAAlignImpl> {
- using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
- AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
-};
-} // namespace
-
-/// ------------------ Function No-Return Attribute ----------------------------
-namespace {
-struct AANoReturnImpl : public AANoReturn {
- AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- bool IsKnown;
- assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
- A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
- (void)IsKnown;
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "noreturn" : "may-return";
- }
-
- /// See AbstractAttribute::updateImpl(Attributor &A).
- ChangeStatus updateImpl(Attributor &A) override {
- auto CheckForNoReturn = [](Instruction &) { return false; };
- bool UsedAssumedInformation = false;
- if (!A.checkForAllInstructions(CheckForNoReturn, *this,
- {(unsigned)Instruction::Ret},
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
- return ChangeStatus::UNCHANGED;
- }
-};
-
-struct AANoReturnFunction final : AANoReturnImpl {
- AANoReturnFunction(const IRPosition &IRP, Attributor &A)
- : AANoReturnImpl(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
-};
-
-/// NoReturn attribute deduction for a call sites.
-struct AANoReturnCallSite final
- : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
- AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
-};
-} // namespace
-
-/// ----------------------- Instance Info ---------------------------------
-
-namespace {
-/// A class to hold the state of for no-capture attributes.
-struct AAInstanceInfoImpl : public AAInstanceInfo {
- AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
- : AAInstanceInfo(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- Value &V = getAssociatedValue();
- if (auto *C = dyn_cast<Constant>(&V)) {
- if (C->isThreadDependent())
- indicatePessimisticFixpoint();
- else
- indicateOptimisticFixpoint();
- return;
- }
- if (auto *CB = dyn_cast<CallBase>(&V))
- if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
- !CB->mayReadFromMemory()) {
- indicateOptimisticFixpoint();
- return;
- }
- if (auto *I = dyn_cast<Instruction>(&V)) {
- const auto *CI =
- A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
- *I->getFunction());
- if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
- indicatePessimisticFixpoint();
- return;
- }
- }
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
-
- Value &V = getAssociatedValue();
- const Function *Scope = nullptr;
- if (auto *I = dyn_cast<Instruction>(&V))
- Scope = I->getFunction();
- if (auto *A = dyn_cast<Argument>(&V)) {
- Scope = A->getParent();
- if (!Scope->hasLocalLinkage())
- return Changed;
- }
- if (!Scope)
- return indicateOptimisticFixpoint();
-
- bool IsKnownNoRecurse;
- if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
- A, this, IRPosition::function(*Scope), DepClassTy::OPTIONAL,
- IsKnownNoRecurse))
- return Changed;
-
- auto UsePred = [&](const Use &U, bool &Follow) {
- const Instruction *UserI = dyn_cast<Instruction>(U.getUser());
- if (!UserI || isa<GetElementPtrInst>(UserI) || isa<CastInst>(UserI) ||
- isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
- Follow = true;
- return true;
- }
- if (isa<LoadInst>(UserI) || isa<CmpInst>(UserI) ||
- (isa<StoreInst>(UserI) &&
- cast<StoreInst>(UserI)->getValueOperand() != U.get()))
- return true;
- if (auto *CB = dyn_cast<CallBase>(UserI)) {
- // This check is not guaranteeing uniqueness but for now that we cannot
- // end up with two versions of \p U thinking it was one.
- auto *Callee = dyn_cast_if_present<Function>(CB->getCalledOperand());
- if (!Callee || !Callee->hasLocalLinkage())
- return true;
- if (!CB->isArgOperand(&U))
- return false;
- const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
- *this, IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U)),
- DepClassTy::OPTIONAL);
- if (!ArgInstanceInfoAA ||
- !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
- return false;
- // If this call base might reach the scope again we might forward the
- // argument back here. This is very conservative.
- if (AA::isPotentiallyReachable(
- A, *CB, *Scope, *this, /* ExclusionSet */ nullptr,
- [Scope](const Function &Fn) { return &Fn != Scope; }))
- return false;
- return true;
- }
- return false;
- };
-
- auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
- if (auto *SI = dyn_cast<StoreInst>(OldU.getUser())) {
- auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
- if ((isa<AllocaInst>(Ptr) || isNoAliasCall(Ptr)) &&
- AA::isDynamicallyUnique(A, *this, *Ptr))
- return true;
- }
- return false;
- };
-
- if (!A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ true,
- DepClassTy::OPTIONAL,
- /* IgnoreDroppableUses */ true, EquivalentUseCB))
- return indicatePessimisticFixpoint();
-
- return Changed;
- }
-
- /// See AbstractState::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {}
-};
-
-/// InstanceInfo attribute for floating values.
-struct AAInstanceInfoFloating : AAInstanceInfoImpl {
- AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
- : AAInstanceInfoImpl(IRP, A) {}
-};
-
-/// NoCapture attribute for function arguments.
-struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
- AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
- : AAInstanceInfoFloating(IRP, A) {}
-};
-
-/// InstanceInfo attribute for call site arguments.
-struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
- AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAInstanceInfoImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // TODO: Once we have call site specific value information we can provide
- // call site specific liveness information and then it makes
- // sense to specialize attributes for call sites arguments instead of
- // redirecting requests to the callee argument.
- Argument *Arg = getAssociatedArgument();
- if (!Arg)
- return indicatePessimisticFixpoint();
- const IRPosition &ArgPos = IRPosition::argument(*Arg);
- auto *ArgAA =
- A.getAAFor<AAInstanceInfo>(*this, ArgPos, DepClassTy::REQUIRED);
- if (!ArgAA)
- return indicatePessimisticFixpoint();
- return clampStateAndIndicateChange(getState(), ArgAA->getState());
- }
-};
-
-/// InstanceInfo attribute for function return value.
-struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
- AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
- : AAInstanceInfoImpl(IRP, A) {
- llvm_unreachable("InstanceInfo is not applicable to function returns!");
- }
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- llvm_unreachable("InstanceInfo is not applicable to function returns!");
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- llvm_unreachable("InstanceInfo is not applicable to function returns!");
- }
-};
-
-/// InstanceInfo attribute deduction for a call site return value.
-struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
- AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AAInstanceInfoFloating(IRP, A) {}
-};
-} // namespace
-
-/// ----------------------- Variable Capturing ---------------------------------
-bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions) {
- assert(ImpliedAttributeKind == Attribute::NoCapture &&
- "Unexpected attribute kind");
- Value &V = IRP.getAssociatedValue();
- if (!IRP.isArgumentPosition())
- return V.use_empty();
-
- // You cannot "capture" null in the default address space.
- if (isa<UndefValue>(V) || (isa<ConstantPointerNull>(V) &&
- V.getType()->getPointerAddressSpace() == 0)) {
- return true;
- }
-
- if (A.hasAttr(IRP, {Attribute::NoCapture},
- /* IgnoreSubsumingPositions */ true, Attribute::NoCapture))
- return true;
-
- if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
- if (Argument *Arg = IRP.getAssociatedArgument())
- if (A.hasAttr(IRPosition::argument(*Arg),
- {Attribute::NoCapture, Attribute::ByVal},
- /* IgnoreSubsumingPositions */ true)) {
- A.manifestAttrs(IRP,
- Attribute::get(V.getContext(), Attribute::NoCapture));
- return true;
- }
-
- if (const Function *F = IRP.getAssociatedFunction()) {
- // Check what state the associated function can actually capture.
- AANoCapture::StateType State;
- determineFunctionCaptureCapabilities(IRP, *F, State);
- if (State.isKnown(NO_CAPTURE)) {
- A.manifestAttrs(IRP,
- Attribute::get(V.getContext(), Attribute::NoCapture));
- return true;
- }
- }
-
- return false;
-}
-
-/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
-/// depending on the ability of the function associated with \p IRP to capture
-/// state in memory and through "returning/throwing", respectively.
-void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP,
- const Function &F,
- BitIntegerState &State) {
- // TODO: Once we have memory behavior attributes we should use them here.
-
- // If we know we cannot communicate or write to memory, we do not care about
- // ptr2int anymore.
- bool ReadOnly = F.onlyReadsMemory();
- bool NoThrow = F.doesNotThrow();
- bool IsVoidReturn = F.getReturnType()->isVoidTy();
- if (ReadOnly && NoThrow && IsVoidReturn) {
- State.addKnownBits(NO_CAPTURE);
- return;
- }
-
- // A function cannot capture state in memory if it only reads memory, it can
- // however return/throw state and the state might be influenced by the
- // pointer value, e.g., loading from a returned pointer might reveal a bit.
- if (ReadOnly)
- State.addKnownBits(NOT_CAPTURED_IN_MEM);
-
- // A function cannot communicate state back if it does not through
- // exceptions and doesn not return values.
- if (NoThrow && IsVoidReturn)
- State.addKnownBits(NOT_CAPTURED_IN_RET);
-
- // Check existing "returned" attributes.
- int ArgNo = IRP.getCalleeArgNo();
- if (!NoThrow || ArgNo < 0 ||
- !F.getAttributes().hasAttrSomewhere(Attribute::Returned))
- return;
-
- for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
- if (F.hasParamAttribute(U, Attribute::Returned)) {
- if (U == unsigned(ArgNo))
- State.removeAssumedBits(NOT_CAPTURED_IN_RET);
- else if (ReadOnly)
- State.addKnownBits(NO_CAPTURE);
- else
- State.addKnownBits(NOT_CAPTURED_IN_RET);
- break;
- }
-}
-
-namespace {
-/// A class to hold the state of for no-capture attributes.
-struct AANoCaptureImpl : public AANoCapture {
- AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- bool IsKnown;
- assert(!AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
- (void)IsKnown;
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override;
-
- /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).
- void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
- SmallVectorImpl<Attribute> &Attrs) const override {
- if (!isAssumedNoCaptureMaybeReturned())
- return;
-
- if (isArgumentPosition()) {
- if (isAssumedNoCapture())
- Attrs.emplace_back(Attribute::get(Ctx, Attribute::NoCapture));
- else if (ManifestInternal)
- Attrs.emplace_back(Attribute::get(Ctx, "no-capture-maybe-returned"));
- }
- }
-
- /// See AbstractState::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- if (isKnownNoCapture())
- return "known not-captured";
- if (isAssumedNoCapture())
- return "assumed not-captured";
- if (isKnownNoCaptureMaybeReturned())
- return "known not-captured-maybe-returned";
- if (isAssumedNoCaptureMaybeReturned())
- return "assumed not-captured-maybe-returned";
- return "assumed-captured";
- }
-
- /// Check the use \p U and update \p State accordingly. Return true if we
- /// should continue to update the state.
- bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
- bool &Follow) {
- Instruction *UInst = cast<Instruction>(U.getUser());
- LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
- << *UInst << "\n");
-
- // Deal with ptr2int by following uses.
- if (isa<PtrToIntInst>(UInst)) {
- LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
- return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
- /* Return */ true);
- }
-
- // For stores we already checked if we can follow them, if they make it
- // here we give up.
- if (isa<StoreInst>(UInst))
- return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
- /* Return */ true);
-
- // Explicitly catch return instructions.
- if (isa<ReturnInst>(UInst)) {
- if (UInst->getFunction() == getAnchorScope())
- return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
- /* Return */ true);
- return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
- /* Return */ true);
- }
-
- // For now we only use special logic for call sites. However, the tracker
- // itself knows about a lot of other non-capturing cases already.
- auto *CB = dyn_cast<CallBase>(UInst);
- if (!CB || !CB->isArgOperand(&U))
- return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
- /* Return */ true);
-
- unsigned ArgNo = CB->getArgOperandNo(&U);
- const IRPosition &CSArgPos = IRPosition::callsite_argument(*CB, ArgNo);
- // If we have a abstract no-capture attribute for the argument we can use
- // it to justify a non-capture attribute here. This allows recursion!
- bool IsKnownNoCapture;
- const AANoCapture *ArgNoCaptureAA = nullptr;
- bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, CSArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
- &ArgNoCaptureAA);
- if (IsAssumedNoCapture)
- return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
- /* Return */ false);
- if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
- Follow = true;
- return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
- /* Return */ false);
- }
-
- // Lastly, we could not find a reason no-capture can be assumed so we don't.
- return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
- /* Return */ true);
- }
-
- /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
- /// \p CapturedInRet, then return true if we should continue updating the
- /// state.
- static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
- bool CapturedInInt, bool CapturedInRet) {
- LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
- << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
- if (CapturedInMem)
- State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_MEM);
- if (CapturedInInt)
- State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_INT);
- if (CapturedInRet)
- State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_RET);
- return State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
- }
-};
-
-ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
- const IRPosition &IRP = getIRPosition();
- Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
- : &IRP.getAssociatedValue();
- if (!V)
- return indicatePessimisticFixpoint();
-
- const Function *F =
- isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
- assert(F && "Expected a function!");
- const IRPosition &FnPos = IRPosition::function(*F);
-
- AANoCapture::StateType T;
-
- // Readonly means we cannot capture through memory.
- bool IsKnown;
- if (AA::isAssumedReadOnly(A, FnPos, *this, IsKnown)) {
- T.addKnownBits(NOT_CAPTURED_IN_MEM);
- if (IsKnown)
- addKnownBits(NOT_CAPTURED_IN_MEM);
- }
-
- // Make sure all returned values are different than the underlying value.
- // TODO: we could do this in a more sophisticated way inside
- // AAReturnedValues, e.g., track all values that escape through returns
- // directly somehow.
- auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
- SmallVector<AA::ValueAndContext> Values;
- if (!A.getAssumedSimplifiedValues(IRPosition::returned(*F), this, Values,
- AA::ValueScope::Intraprocedural,
- UsedAssumedInformation))
- return false;
- bool SeenConstant = false;
- for (const AA::ValueAndContext &VAC : Values) {
- if (isa<Constant>(VAC.getValue())) {
- if (SeenConstant)
- return false;
- SeenConstant = true;
- } else if (!isa<Argument>(VAC.getValue()) ||
- VAC.getValue() == getAssociatedArgument())
- return false;
- }
- return true;
- };
-
- bool IsKnownNoUnwind;
- if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
- A, this, FnPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
- bool IsVoidTy = F->getReturnType()->isVoidTy();
- bool UsedAssumedInformation = false;
- if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
- T.addKnownBits(NOT_CAPTURED_IN_RET);
- if (T.isKnown(NOT_CAPTURED_IN_MEM))
- return ChangeStatus::UNCHANGED;
- if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
- addKnownBits(NOT_CAPTURED_IN_RET);
- if (isKnown(NOT_CAPTURED_IN_MEM))
- return indicateOptimisticFixpoint();
- }
- }
- }
-
- auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
- const auto *DerefAA = A.getAAFor<AADereferenceable>(
- *this, IRPosition::value(*O), DepClassTy::OPTIONAL);
- return DerefAA && DerefAA->getAssumedDereferenceableBytes();
- };
-
- auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
- switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) {
- case UseCaptureKind::NO_CAPTURE:
- return true;
- case UseCaptureKind::MAY_CAPTURE:
- return checkUse(A, T, U, Follow);
- case UseCaptureKind::PASSTHROUGH:
- Follow = true;
- return true;
- }
- llvm_unreachable("Unexpected use capture kind!");
- };
-
- if (!A.checkForAllUses(UseCheck, *this, *V))
- return indicatePessimisticFixpoint();
-
- AANoCapture::StateType &S = getState();
- auto Assumed = S.getAssumed();
- S.intersectAssumedBits(T.getAssumed());
- if (!isAssumedNoCaptureMaybeReturned())
- return indicatePessimisticFixpoint();
- return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
-}
-
-/// NoCapture attribute for function arguments.
-struct AANoCaptureArgument final : AANoCaptureImpl {
- AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
- : AANoCaptureImpl(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
-};
-
-/// NoCapture attribute for call site arguments.
-struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
- AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AANoCaptureImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // TODO: Once we have call site specific value information we can provide
- // call site specific liveness information and then it makes
- // sense to specialize attributes for call sites arguments instead of
- // redirecting requests to the callee argument.
- Argument *Arg = getAssociatedArgument();
- if (!Arg)
- return indicatePessimisticFixpoint();
- const IRPosition &ArgPos = IRPosition::argument(*Arg);
- bool IsKnownNoCapture;
- const AANoCapture *ArgAA = nullptr;
- if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, ArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
- &ArgAA))
- return ChangeStatus::UNCHANGED;
- if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
- return indicatePessimisticFixpoint();
- return clampStateAndIndicateChange(getState(), ArgAA->getState());
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nocapture)};
-};
-
-/// NoCapture attribute for floating values.
-struct AANoCaptureFloating final : AANoCaptureImpl {
- AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
- : AANoCaptureImpl(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(nocapture)
- }
-};
-
-/// NoCapture attribute for function return value.
-struct AANoCaptureReturned final : AANoCaptureImpl {
- AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
- : AANoCaptureImpl(IRP, A) {
- llvm_unreachable("NoCapture is not applicable to function returns!");
- }
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- llvm_unreachable("NoCapture is not applicable to function returns!");
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- llvm_unreachable("NoCapture is not applicable to function returns!");
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {}
-};
-
-/// NoCapture attribute deduction for a call site return value.
-struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
- AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AANoCaptureImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- const Function *F = getAnchorScope();
- // Check what state the associated function can actually capture.
- determineFunctionCaptureCapabilities(getIRPosition(), *F, *this);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSRET_ATTR(nocapture)
- }
-};
-} // namespace
-
-/// ------------------ Value Simplify Attribute ----------------------------
-
-bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
- // FIXME: Add a typecast support.
- SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
- SimplifiedAssociatedValue, Other, Ty);
- if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
- return false;
-
- LLVM_DEBUG({
- if (SimplifiedAssociatedValue)
- dbgs() << "[ValueSimplify] is assumed to be "
- << **SimplifiedAssociatedValue << "\n";
- else
- dbgs() << "[ValueSimplify] is assumed to be <none>\n";
- });
- return true;
-}
-
-namespace {
-struct AAValueSimplifyImpl : AAValueSimplify {
- AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
- : AAValueSimplify(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- if (getAssociatedValue().getType()->isVoidTy())
- indicatePessimisticFixpoint();
- if (A.hasSimplificationCallback(getIRPosition()))
- indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- LLVM_DEBUG({
- dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
- if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
- dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
- });
- return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
- : "not-simple";
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {}
-
- /// See AAValueSimplify::getAssumedSimplifiedValue()
- std::optional<Value *>
- getAssumedSimplifiedValue(Attributor &A) const override {
- return SimplifiedAssociatedValue;
- }
-
- /// Ensure the return value is \p V with type \p Ty, if not possible return
- /// nullptr. If \p Check is true we will only verify such an operation would
- /// suceed and return a non-nullptr value if that is the case. No IR is
- /// generated or modified.
- static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
- bool Check) {
- if (auto *TypedV = AA::getWithType(V, Ty))
- return TypedV;
- if (CtxI && V.getType()->canLosslesslyBitCastTo(&Ty))
- return Check ? &V
- : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
- &V, &Ty, "", CtxI->getIterator());
- return nullptr;
- }
-
- /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
- /// If \p Check is true we will only verify such an operation would suceed and
- /// return a non-nullptr value if that is the case. No IR is generated or
- /// modified.
- static Value *reproduceInst(Attributor &A,
- const AbstractAttribute &QueryingAA,
- Instruction &I, Type &Ty, Instruction *CtxI,
- bool Check, ValueToValueMapTy &VMap) {
- assert(CtxI && "Cannot reproduce an instruction without context!");
- if (Check && (I.mayReadFromMemory() ||
- !isSafeToSpeculativelyExecute(&I, CtxI, /* DT */ nullptr,
- /* TLI */ nullptr)))
- return nullptr;
- for (Value *Op : I.operands()) {
- Value *NewOp = reproduceValue(A, QueryingAA, *Op, Ty, CtxI, Check, VMap);
- if (!NewOp) {
- assert(Check && "Manifest of new value unexpectedly failed!");
- return nullptr;
- }
- if (!Check)
- VMap[Op] = NewOp;
- }
- if (Check)
- return &I;
-
- Instruction *CloneI = I.clone();
- // TODO: Try to salvage debug information here.
- CloneI->setDebugLoc(DebugLoc());
- VMap[&I] = CloneI;
- CloneI->insertBefore(CtxI);
- RemapInstruction(CloneI, VMap);
- return CloneI;
- }
-
- /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
- /// If \p Check is true we will only verify such an operation would suceed and
- /// return a non-nullptr value if that is the case. No IR is generated or
- /// modified.
- static Value *reproduceValue(Attributor &A,
- const AbstractAttribute &QueryingAA, Value &V,
- Type &Ty, Instruction *CtxI, bool Check,
- ValueToValueMapTy &VMap) {
- if (const auto &NewV = VMap.lookup(&V))
- return NewV;
- bool UsedAssumedInformation = false;
- std::optional<Value *> SimpleV = A.getAssumedSimplified(
- V, QueryingAA, UsedAssumedInformation, AA::Interprocedural);
- if (!SimpleV.has_value())
- return PoisonValue::get(&Ty);
- Value *EffectiveV = &V;
- if (*SimpleV)
- EffectiveV = *SimpleV;
- if (auto *C = dyn_cast<Constant>(EffectiveV))
- return C;
- if (CtxI && AA::isValidAtPosition(AA::ValueAndContext(*EffectiveV, *CtxI),
- A.getInfoCache()))
- return ensureType(A, *EffectiveV, Ty, CtxI, Check);
- if (auto *I = dyn_cast<Instruction>(EffectiveV))
- if (Value *NewV = reproduceInst(A, QueryingAA, *I, Ty, CtxI, Check, VMap))
- return ensureType(A, *NewV, Ty, CtxI, Check);
- return nullptr;
- }
-
- /// Return a value we can use as replacement for the associated one, or
- /// nullptr if we don't have one that makes sense.
- Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
- Value *NewV = SimplifiedAssociatedValue
- ? *SimplifiedAssociatedValue
- : UndefValue::get(getAssociatedType());
- if (NewV && NewV != &getAssociatedValue()) {
- ValueToValueMapTy VMap;
- // First verify we can reprduce the value with the required type at the
- // context location before we actually start modifying the IR.
- if (reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,
- /* CheckOnly */ true, VMap))
- return reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,
- /* CheckOnly */ false, VMap);
- }
- return nullptr;
- }
-
- /// Helper function for querying AAValueSimplify and updating candidate.
- /// \param IRP The value position we are trying to unify with SimplifiedValue
- bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
- const IRPosition &IRP, bool Simplify = true) {
- bool UsedAssumedInformation = false;
- std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
- if (Simplify)
- QueryingValueSimplified = A.getAssumedSimplified(
- IRP, QueryingAA, UsedAssumedInformation, AA::Interprocedural);
- return unionAssumed(QueryingValueSimplified);
- }
-
- /// Returns a candidate is found or not
- template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
- if (!getAssociatedValue().getType()->isIntegerTy())
- return false;
-
- // This will also pass the call base context.
- const auto *AA =
- A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
- if (!AA)
- return false;
-
- std::optional<Constant *> COpt = AA->getAssumedConstant(A);
-
- if (!COpt) {
- SimplifiedAssociatedValue = std::nullopt;
- A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);
- return true;
- }
- if (auto *C = *COpt) {
- SimplifiedAssociatedValue = C;
- A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);
- return true;
- }
- return false;
- }
-
- bool askSimplifiedValueForOtherAAs(Attributor &A) {
- if (askSimplifiedValueFor<AAValueConstantRange>(A))
- return true;
- if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
- return true;
- return false;
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- for (auto &U : getAssociatedValue().uses()) {
- // Check if we need to adjust the insertion point to make sure the IR is
- // valid.
- Instruction *IP = dyn_cast<Instruction>(U.getUser());
- if (auto *PHI = dyn_cast_or_null<PHINode>(IP))
- IP = PHI->getIncomingBlock(U)->getTerminator();
- if (auto *NewV = manifestReplacementValue(A, IP)) {
- LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
- << " -> " << *NewV << " :: " << *this << "\n");
- if (A.changeUseAfterManifest(U, *NewV))
- Changed = ChangeStatus::CHANGED;
- }
- }
-
- return Changed | AAValueSimplify::manifest(A);
- }
-
- /// See AbstractState::indicatePessimisticFixpoint(...).
- ChangeStatus indicatePessimisticFixpoint() override {
- SimplifiedAssociatedValue = &getAssociatedValue();
- return AAValueSimplify::indicatePessimisticFixpoint();
- }
-};
-
-struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
- AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
- : AAValueSimplifyImpl(IRP, A) {}
-
- void initialize(Attributor &A) override {
- AAValueSimplifyImpl::initialize(A);
- if (A.hasAttr(getIRPosition(),
- {Attribute::InAlloca, Attribute::Preallocated,
- Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
- /* IgnoreSubsumingPositions */ true))
- indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // Byval is only replacable if it is readonly otherwise we would write into
- // the replaced value and not the copy that byval creates implicitly.
- Argument *Arg = getAssociatedArgument();
- if (Arg->hasByValAttr()) {
- // TODO: We probably need to verify synchronization is not an issue, e.g.,
- // there is no race by not copying a constant byval.
- bool IsKnown;
- if (!AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
- return indicatePessimisticFixpoint();
- }
-
- auto Before = SimplifiedAssociatedValue;
-
- auto PredForCallSite = [&](AbstractCallSite ACS) {
- const IRPosition &ACSArgPos =
- IRPosition::callsite_argument(ACS, getCallSiteArgNo());
- // Check if a coresponding argument was found or if it is on not
- // associated (which can happen for callback calls).
- if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
- return false;
-
- // Simplify the argument operand explicitly and check if the result is
- // valid in the current scope. This avoids refering to simplified values
- // in other functions, e.g., we don't want to say a an argument in a
- // static function is actually an argument in a different function.
- bool UsedAssumedInformation = false;
- std::optional<Constant *> SimpleArgOp =
- A.getAssumedConstant(ACSArgPos, *this, UsedAssumedInformation);
- if (!SimpleArgOp)
- return true;
- if (!*SimpleArgOp)
- return false;
- if (!AA::isDynamicallyUnique(A, *this, **SimpleArgOp))
- return false;
- return unionAssumed(*SimpleArgOp);
- };
-
- // Generate a answer specific to a call site context.
- bool Success;
- bool UsedAssumedInformation = false;
- if (hasCallBaseContext() &&
- getCallBaseContext()->getCalledOperand() == Arg->getParent())
- Success = PredForCallSite(
- AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
- else
- Success = A.checkForAllCallSites(PredForCallSite, *this, true,
- UsedAssumedInformation);
-
- if (!Success)
- if (!askSimplifiedValueForOtherAAs(A))
- return indicatePessimisticFixpoint();
-
- // If a candidate was found in this update, return CHANGED.
- return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
- : ChangeStatus ::CHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_ARG_ATTR(value_simplify)
- }
-};
-
-struct AAValueSimplifyReturned : AAValueSimplifyImpl {
- AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
- : AAValueSimplifyImpl(IRP, A) {}
-
- /// See AAValueSimplify::getAssumedSimplifiedValue()
- std::optional<Value *>
- getAssumedSimplifiedValue(Attributor &A) const override {
- if (!isValidState())
- return nullptr;
- return SimplifiedAssociatedValue;
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto Before = SimplifiedAssociatedValue;
-
- auto ReturnInstCB = [&](Instruction &I) {
- auto &RI = cast<ReturnInst>(I);
- return checkAndUpdate(
- A, *this,
- IRPosition::value(*RI.getReturnValue(), getCallBaseContext()));
- };
-
- bool UsedAssumedInformation = false;
- if (!A.checkForAllInstructions(ReturnInstCB, *this, {Instruction::Ret},
- UsedAssumedInformation))
- if (!askSimplifiedValueForOtherAAs(A))
- return indicatePessimisticFixpoint();
-
- // If a candidate was found in this update, return CHANGED.
- return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
- : ChangeStatus ::CHANGED;
- }
-
- ChangeStatus manifest(Attributor &A) override {
- // We queried AAValueSimplify for the returned values so they will be
- // replaced if a simplified form was found. Nothing to do here.
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FNRET_ATTR(value_simplify)
- }
-};
-
-struct AAValueSimplifyFloating : AAValueSimplifyImpl {
- AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
- : AAValueSimplifyImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- AAValueSimplifyImpl::initialize(A);
- Value &V = getAnchorValue();
-
- // TODO: add other stuffs
- if (isa<Constant>(V))
- indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto Before = SimplifiedAssociatedValue;
- if (!askSimplifiedValueForOtherAAs(A))
- return indicatePessimisticFixpoint();
-
- // If a candidate was found in this update, return CHANGED.
- return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
- : ChangeStatus ::CHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
- }
-};
-
-struct AAValueSimplifyFunction : AAValueSimplifyImpl {
- AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
- : AAValueSimplifyImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- SimplifiedAssociatedValue = nullptr;
- indicateOptimisticFixpoint();
- }
- /// See AbstractAttribute::initialize(...).
- ChangeStatus updateImpl(Attributor &A) override {
- llvm_unreachable(
- "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
- }
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FN_ATTR(value_simplify)
- }
-};
-
-struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
- AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
- : AAValueSimplifyFunction(IRP, A) {}
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CS_ATTR(value_simplify)
- }
-};
-
-struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
- AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AAValueSimplifyImpl(IRP, A) {}
-
- void initialize(Attributor &A) override {
- AAValueSimplifyImpl::initialize(A);
- Function *Fn = getAssociatedFunction();
- assert(Fn && "Did expect an associted function");
- for (Argument &Arg : Fn->args()) {
- if (Arg.hasReturnedAttr()) {
- auto IRP = IRPosition::callsite_argument(*cast<CallBase>(getCtxI()),
- Arg.getArgNo());
- if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT &&
- checkAndUpdate(A, *this, IRP))
- indicateOptimisticFixpoint();
- else
- indicatePessimisticFixpoint();
- return;
- }
- }
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- return indicatePessimisticFixpoint();
- }
-
- void trackStatistics() const override {
- STATS_DECLTRACK_CSRET_ATTR(value_simplify)
- }
-};
-
-struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
- AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAValueSimplifyFloating(IRP, A) {}
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- // TODO: We should avoid simplification duplication to begin with.
- auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
- IRPosition::value(getAssociatedValue()), this, DepClassTy::NONE);
- if (FloatAA && FloatAA->getState().isValidState())
- return Changed;
-
- if (auto *NewV = manifestReplacementValue(A, getCtxI())) {
- Use &U = cast<CallBase>(&getAnchorValue())
- ->getArgOperandUse(getCallSiteArgNo());
- if (A.changeUseAfterManifest(U, *NewV))
- Changed = ChangeStatus::CHANGED;
- }
-
- return Changed | AAValueSimplify::manifest(A);
- }
-
- void trackStatistics() const override {
- STATS_DECLTRACK_CSARG_ATTR(value_simplify)
- }
-};
-} // namespace
-
-/// ----------------------- Heap-To-Stack Conversion ---------------------------
-namespace {
-struct AAHeapToStackFunction final : public AAHeapToStack {
-
- struct AllocationInfo {
- /// The call that allocates the memory.
- CallBase *const CB;
-
- /// The library function id for the allocation.
- LibFunc LibraryFunctionId = NotLibFunc;
-
- /// The status wrt. a rewrite.
- enum {
- STACK_DUE_TO_USE,
- STACK_DUE_TO_FREE,
- INVALID,
- } Status = STACK_DUE_TO_USE;
-
- /// Flag to indicate if we encountered a use that might free this allocation
- /// but which is not in the deallocation infos.
- bool HasPotentiallyFreeingUnknownUses = false;
-
- /// Flag to indicate that we should place the new alloca in the function
- /// entry block rather than where the call site (CB) is.
- bool MoveAllocaIntoEntry = true;
-
- /// The set of free calls that use this allocation.
- SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
- };
-
- struct DeallocationInfo {
- /// The call that deallocates the memory.
- CallBase *const CB;
- /// The value freed by the call.
- Value *FreedOp;
-
- /// Flag to indicate if we don't know all objects this deallocation might
- /// free.
- bool MightFreeUnknownObjects = false;
-
- /// The set of allocation calls that are potentially freed.
- SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
- };
-
- AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
- : AAHeapToStack(IRP, A) {}
-
- ~AAHeapToStackFunction() {
- // Ensure we call the destructor so we release any memory allocated in the
- // sets.
- for (auto &It : AllocationInfos)
- It.second->~AllocationInfo();
- for (auto &It : DeallocationInfos)
- It.second->~DeallocationInfo();
- }
-
- void initialize(Attributor &A) override {
- AAHeapToStack::initialize(A);
-
- const Function *F = getAnchorScope();
- const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
-
- auto AllocationIdentifierCB = [&](Instruction &I) {
- CallBase *CB = dyn_cast<CallBase>(&I);
- if (!CB)
- return true;
- if (Value *FreedOp = getFreedOperand(CB, TLI)) {
- DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{CB, FreedOp};
- return true;
- }
- // To do heap to stack, we need to know that the allocation itself is
- // removable once uses are rewritten, and that we can initialize the
- // alloca to the same pattern as the original allocation result.
- if (isRemovableAlloc(CB, TLI)) {
- auto *I8Ty = Type::getInt8Ty(CB->getParent()->getContext());
- if (nullptr != getInitialValueOfAllocation(CB, TLI, I8Ty)) {
- AllocationInfo *AI = new (A.Allocator) AllocationInfo{CB};
- AllocationInfos[CB] = AI;
- if (TLI)
- TLI->getLibFunc(*CB, AI->LibraryFunctionId);
- }
- }
- return true;
- };
-
- bool UsedAssumedInformation = false;
- bool Success = A.checkForAllCallLikeInstructions(
- AllocationIdentifierCB, *this, UsedAssumedInformation,
- /* CheckBBLivenessOnly */ false,
- /* CheckPotentiallyDead */ true);
- (void)Success;
- assert(Success && "Did not expect the call base visit callback to fail!");
-
- Attributor::SimplifictionCallbackTy SCB =
- [](const IRPosition &, const AbstractAttribute *,
- bool &) -> std::optional<Value *> { return nullptr; };
- for (const auto &It : AllocationInfos)
- A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),
- SCB);
- for (const auto &It : DeallocationInfos)
- A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),
- SCB);
- }
-
- const std::string getAsStr(Attributor *A) const override {
- unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
- for (const auto &It : AllocationInfos) {
- if (It.second->Status == AllocationInfo::INVALID)
- ++NumInvalidMallocs;
- else
- ++NumH2SMallocs;
- }
- return "[H2S] Mallocs Good/Bad: " + std::to_string(NumH2SMallocs) + "/" +
- std::to_string(NumInvalidMallocs);
- }
-
- /// See AbstractAttribute::trackStatistics().
- void trackStatistics() const override {
- STATS_DECL(
- MallocCalls, Function,
- "Number of malloc/calloc/aligned_alloc calls converted to allocas");
- for (const auto &It : AllocationInfos)
- if (It.second->Status != AllocationInfo::INVALID)
- ++BUILD_STAT_NAME(MallocCalls, Function);
- }
-
- bool isAssumedHeapToStack(const CallBase &CB) const override {
- if (isValidState())
- if (AllocationInfo *AI =
- AllocationInfos.lookup(const_cast<CallBase *>(&CB)))
- return AI->Status != AllocationInfo::INVALID;
- return false;
- }
-
- bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
- if (!isValidState())
- return false;
-
- for (const auto &It : AllocationInfos) {
- AllocationInfo &AI = *It.second;
- if (AI.Status == AllocationInfo::INVALID)
- continue;
-
- if (AI.PotentialFreeCalls.count(&CB))
- return true;
- }
-
- return false;
- }
-
- ChangeStatus manifest(Attributor &A) override {
- assert(getState().isValidState() &&
- "Attempted to manifest an invalid state!");
-
- ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
- Function *F = getAnchorScope();
- const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
-
- for (auto &It : AllocationInfos) {
- AllocationInfo &AI = *It.second;
- if (AI.Status == AllocationInfo::INVALID)
- continue;
-
- for (CallBase *FreeCall : AI.PotentialFreeCalls) {
- LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
- A.deleteAfterManifest(*FreeCall);
- HasChanged = ChangeStatus::CHANGED;
- }
-
- LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
- << "\n");
-
- auto Remark = [&](OptimizationRemark OR) {
- LibFunc IsAllocShared;
- if (TLI->getLibFunc(*AI.CB, IsAllocShared))
- if (IsAllocShared == LibFunc___kmpc_alloc_shared)
- return OR << "Moving globalized variable to the stack.";
- return OR << "Moving memory allocation from the heap to the stack.";
- };
- if (AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
- A.emitRemark<OptimizationRemark>(AI.CB, "OMP110", Remark);
- else
- A.emitRemark<OptimizationRemark>(AI.CB, "HeapToStack", Remark);
-
- const DataLayout &DL = A.getInfoCache().getDL();
- Value *Size;
- std::optional<APInt> SizeAPI = getSize(A, *this, AI);
- if (SizeAPI) {
- Size = ConstantInt::get(AI.CB->getContext(), *SizeAPI);
- } else {
- LLVMContext &Ctx = AI.CB->getContext();
- ObjectSizeOpts Opts;
- ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts);
- SizeOffsetValue SizeOffsetPair = Eval.compute(AI.CB);
- assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
- cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
- Size = SizeOffsetPair.Size;
- }
-
- BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
- ? F->getEntryBlock().begin()
- : AI.CB->getIterator();
-
- Align Alignment(1);
- if (MaybeAlign RetAlign = AI.CB->getRetAlign())
- Alignment = std::max(Alignment, *RetAlign);
- if (Value *Align = getAllocAlignment(AI.CB, TLI)) {
- std::optional<APInt> AlignmentAPI = getAPInt(A, *this, *Align);
- assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
- "Expected an alignment during manifest!");
- Alignment =
- std::max(Alignment, assumeAligned(AlignmentAPI->getZExtValue()));
- }
-
- // TODO: Hoist the alloca towards the function entry.
- unsigned AS = DL.getAllocaAddrSpace();
- Instruction *Alloca =
- new AllocaInst(Type::getInt8Ty(F->getContext()), AS, Size, Alignment,
- AI.CB->getName() + ".h2s", IP);
-
- if (Alloca->getType() != AI.CB->getType())
- Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
- Alloca, AI.CB->getType(), "malloc_cast", AI.CB->getIterator());
-
- auto *I8Ty = Type::getInt8Ty(F->getContext());
- auto *InitVal = getInitialValueOfAllocation(AI.CB, TLI, I8Ty);
- assert(InitVal &&
- "Must be able to materialize initial memory state of allocation");
-
- A.changeAfterManifest(IRPosition::inst(*AI.CB), *Alloca);
-
- if (auto *II = dyn_cast<InvokeInst>(AI.CB)) {
- auto *NBB = II->getNormalDest();
- BranchInst::Create(NBB, AI.CB->getParent());
- A.deleteAfterManifest(*AI.CB);
- } else {
- A.deleteAfterManifest(*AI.CB);
- }
-
- // Initialize the alloca with the same value as used by the allocation
- // function. We can skip undef as the initial value of an alloc is
- // undef, and the memset would simply end up being DSEd.
- if (!isa<UndefValue>(InitVal)) {
- IRBuilder<> Builder(Alloca->getNextNode());
- // TODO: Use alignment above if align!=1
- Builder.CreateMemSet(Alloca, InitVal, Size, std::nullopt);
- }
- HasChanged = ChangeStatus::CHANGED;
- }
-
- return HasChanged;
- }
-
- std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
- Value &V) {
- bool UsedAssumedInformation = false;
- std::optional<Constant *> SimpleV =
- A.getAssumedConstant(V, AA, UsedAssumedInformation);
- if (!SimpleV)
- return APInt(64, 0);
- if (auto *CI = dyn_cast_or_null<ConstantInt>(*SimpleV))
- return CI->getValue();
- return std::nullopt;
- }
-
- std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
- AllocationInfo &AI) {
- auto Mapper = [&](const Value *V) -> const Value * {
- bool UsedAssumedInformation = false;
- if (std::optional<Constant *> SimpleV =
- A.getAssumedConstant(*V, AA, UsedAssumedInformation))
- if (*SimpleV)
- return *SimpleV;
- return V;
- };
-
- const Function *F = getAnchorScope();
- const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
- return getAllocSize(AI.CB, TLI, Mapper);
- }
-
- /// Collection of all malloc-like calls in a function with associated
- /// information.
- MapVector<CallBase *, AllocationInfo *> AllocationInfos;
-
- /// Collection of all free-like calls in a function with associated
- /// information.
- MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
-
- ChangeStatus updateImpl(Attributor &A) override;
-};
-
-ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- const Function *F = getAnchorScope();
- const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
-
- const auto *LivenessAA =
- A.getAAFor<AAIsDead>(*this, IRPosition::function(*F), DepClassTy::NONE);
-
- MustBeExecutedContextExplorer *Explorer =
- A.getInfoCache().getMustBeExecutedContextExplorer();
-
- bool StackIsAccessibleByOtherThreads =
- A.getInfoCache().stackIsAccessibleByOtherThreads();
-
- LoopInfo *LI =
- A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(*F);
- std::optional<bool> MayContainIrreducibleControl;
- auto IsInLoop = [&](BasicBlock &BB) {
- if (&F->getEntryBlock() == &BB)
- return false;
- if (!MayContainIrreducibleControl.has_value())
- MayContainIrreducibleControl = mayContainIrreducibleControl(*F, LI);
- if (*MayContainIrreducibleControl)
- return true;
- if (!LI)
- return true;
- return LI->getLoopFor(&BB) != nullptr;
- };
-
- // Flag to ensure we update our deallocation information at most once per
- // updateImpl call and only if we use the free check reasoning.
- bool HasUpdatedFrees = false;
-
- auto UpdateFrees = [&]() {
- HasUpdatedFrees = true;
-
- for (auto &It : DeallocationInfos) {
- DeallocationInfo &DI = *It.second;
- // For now we cannot use deallocations that have unknown inputs, skip
- // them.
- if (DI.MightFreeUnknownObjects)
- continue;
-
- // No need to analyze dead calls, ignore them instead.
- bool UsedAssumedInformation = false;
- if (A.isAssumedDead(*DI.CB, this, LivenessAA, UsedAssumedInformation,
- /* CheckBBLivenessOnly */ true))
- continue;
-
- // Use the non-optimistic version to get the freed object.
- Value *Obj = getUnderlyingObject(DI.FreedOp);
- if (!Obj) {
- LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
- DI.MightFreeUnknownObjects = true;
- continue;
- }
-
- // Free of null and undef can be ignored as no-ops (or UB in the latter
- // case).
- if (isa<ConstantPointerNull>(Obj) || isa<UndefValue>(Obj))
- continue;
-
- CallBase *ObjCB = dyn_cast<CallBase>(Obj);
- if (!ObjCB) {
- LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
- << "\n");
- DI.MightFreeUnknownObjects = true;
- continue;
- }
-
- AllocationInfo *AI = AllocationInfos.lookup(ObjCB);
- if (!AI) {
- LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
- << "\n");
- DI.MightFreeUnknownObjects = true;
- continue;
- }
-
- DI.PotentialAllocationCalls.insert(ObjCB);
- }
- };
-
- auto FreeCheck = [&](AllocationInfo &AI) {
- // If the stack is not accessible by other threads, the "must-free" logic
- // doesn't apply as the pointer could be shared and needs to be places in
- // "shareable" memory.
- if (!StackIsAccessibleByOtherThreads) {
- bool IsKnownNoSycn;
- if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
- A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoSycn)) {
- LLVM_DEBUG(
- dbgs() << "[H2S] found an escaping use, stack is not accessible by "
- "other threads and function is not nosync:\n");
- return false;
- }
- }
- if (!HasUpdatedFrees)
- UpdateFrees();
-
- // TODO: Allow multi exit functions that have different free calls.
- if (AI.PotentialFreeCalls.size() != 1) {
- LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
- << AI.PotentialFreeCalls.size() << "\n");
- return false;
- }
- CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
- DeallocationInfo *DI = DeallocationInfos.lookup(UniqueFree);
- if (!DI) {
- LLVM_DEBUG(
- dbgs() << "[H2S] unique free call was not known as deallocation call "
- << *UniqueFree << "\n");
- return false;
- }
- if (DI->MightFreeUnknownObjects) {
- LLVM_DEBUG(
- dbgs() << "[H2S] unique free call might free unknown allocations\n");
- return false;
- }
- if (DI->PotentialAllocationCalls.empty())
- return true;
- if (DI->PotentialAllocationCalls.size() > 1) {
- LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
- << DI->PotentialAllocationCalls.size()
- << " different allocations\n");
- return false;
- }
- if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
- LLVM_DEBUG(
- dbgs()
- << "[H2S] unique free call not known to free this allocation but "
- << **DI->PotentialAllocationCalls.begin() << "\n");
- return false;
- }
-
- // __kmpc_alloc_shared and __kmpc_alloc_free are by construction matched.
- if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared) {
- Instruction *CtxI = isa<InvokeInst>(AI.CB) ? AI.CB : AI.CB->getNextNode();
- if (!Explorer || !Explorer->findInContextOf(UniqueFree, CtxI)) {
- LLVM_DEBUG(
- dbgs()
- << "[H2S] unique free call might not be executed with the allocation "
- << *UniqueFree << "\n");
- return false;
- }
- }
- return true;
- };
-
- auto UsesCheck = [&](AllocationInfo &AI) {
- bool ValidUsesOnly = true;
-
- auto Pred = [&](const Use &U, bool &Follow) -> bool {
- Instruction *UserI = cast<Instruction>(U.getUser());
- if (isa<LoadInst>(UserI))
- return true;
- if (auto *SI = dyn_cast<StoreInst>(UserI)) {
- if (SI->getValueOperand() == U.get()) {
- LLVM_DEBUG(dbgs()
- << "[H2S] escaping store to memory: " << *UserI << "\n");
- ValidUsesOnly = false;
- } else {
- // A store into the malloc'ed memory is fine.
- }
- return true;
- }
- if (auto *CB = dyn_cast<CallBase>(UserI)) {
- if (!CB->isArgOperand(&U) || CB->isLifetimeStartOrEnd())
- return true;
- if (DeallocationInfos.count(CB)) {
- AI.PotentialFreeCalls.insert(CB);
- return true;
- }
-
- unsigned ArgNo = CB->getArgOperandNo(&U);
- auto CBIRP = IRPosition::callsite_argument(*CB, ArgNo);
-
- bool IsKnownNoCapture;
- bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoCapture);
-
- // If a call site argument use is nofree, we are fine.
- bool IsKnownNoFree;
- bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
- A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoFree);
-
- if (!IsAssumedNoCapture ||
- (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
- !IsAssumedNoFree)) {
- AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
-
- // Emit a missed remark if this is missed OpenMP globalization.
- auto Remark = [&](OptimizationRemarkMissed ORM) {
- return ORM
- << "Could not move globalized variable to the stack. "
- "Variable is potentially captured in call. Mark "
- "parameter as `__attribute__((noescape))` to override.";
- };
-
- if (ValidUsesOnly &&
- AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
- A.emitRemark<OptimizationRemarkMissed>(CB, "OMP113", Remark);
-
- LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
- ValidUsesOnly = false;
- }
- return true;
- }
-
- if (isa<GetElementPtrInst>(UserI) || isa<BitCastInst>(UserI) ||
- isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
- Follow = true;
- return true;
- }
- // Unknown user for which we can not track uses further (in a way that
- // makes sense).
- LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
- ValidUsesOnly = false;
- return true;
- };
- if (!A.checkForAllUses(Pred, *this, *AI.CB, /* CheckBBLivenessOnly */ false,
- DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
- [&](const Use &OldU, const Use &NewU) {
- auto *SI = dyn_cast<StoreInst>(OldU.getUser());
- return !SI || StackIsAccessibleByOtherThreads ||
- AA::isAssumedThreadLocalObject(
- A, *SI->getPointerOperand(), *this);
- }))
- return false;
- return ValidUsesOnly;
- };
-
- // The actual update starts here. We look at all allocations and depending on
- // their status perform the appropriate check(s).
- for (auto &It : AllocationInfos) {
- AllocationInfo &AI = *It.second;
- if (AI.Status == AllocationInfo::INVALID)
- continue;
-
- if (Value *Align = getAllocAlignment(AI.CB, TLI)) {
- std::optional<APInt> APAlign = getAPInt(A, *this, *Align);
- if (!APAlign) {
- // Can't generate an alloca which respects the required alignment
- // on the allocation.
- LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
- << "\n");
- AI.Status = AllocationInfo::INVALID;
- Changed = ChangeStatus::CHANGED;
- continue;
- }
- if (APAlign->ugt(llvm::Value::MaximumAlignment) ||
- !APAlign->isPowerOf2()) {
- LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
- << "\n");
- AI.Status = AllocationInfo::INVALID;
- Changed = ChangeStatus::CHANGED;
- continue;
- }
- }
-
- std::optional<APInt> Size = getSize(A, *this, AI);
- if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
- MaxHeapToStackSize != -1) {
- if (!Size || Size->ugt(MaxHeapToStackSize)) {
- LLVM_DEBUG({
- if (!Size)
- dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
- else
- dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
- << MaxHeapToStackSize << "\n";
- });
-
- AI.Status = AllocationInfo::INVALID;
- Changed = ChangeStatus::CHANGED;
- continue;
- }
- }
-
- switch (AI.Status) {
- case AllocationInfo::STACK_DUE_TO_USE:
- if (UsesCheck(AI))
- break;
- AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
- [[fallthrough]];
- case AllocationInfo::STACK_DUE_TO_FREE:
- if (FreeCheck(AI))
- break;
- AI.Status = AllocationInfo::INVALID;
- Changed = ChangeStatus::CHANGED;
- break;
- case AllocationInfo::INVALID:
- llvm_unreachable("Invalid allocations should never reach this point!");
- };
-
- // Check if we still think we can move it into the entry block. If the
- // alloca comes from a converted __kmpc_alloc_shared then we can usually
- // ignore the potential compilations associated with loops.
- bool IsGlobalizedLocal =
- AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared;
- if (AI.MoveAllocaIntoEntry &&
- (!Size.has_value() ||
- (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
- AI.MoveAllocaIntoEntry = false;
- }
-
- return Changed;
-}
-} // namespace
-
-/// ----------------------- Privatizable Pointers ------------------------------
-namespace {
-struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
- AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
- : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
-
- ChangeStatus indicatePessimisticFixpoint() override {
- AAPrivatizablePtr::indicatePessimisticFixpoint();
- PrivatizableType = nullptr;
- return ChangeStatus::CHANGED;
- }
-
- /// Identify the type we can chose for a private copy of the underlying
- /// argument. std::nullopt means it is not clear yet, nullptr means there is
- /// none.
- virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
-
- /// Return a privatizable type that encloses both T0 and T1.
- /// TODO: This is merely a stub for now as we should manage a mapping as well.
- std::optional<Type *> combineTypes(std::optional<Type *> T0,
- std::optional<Type *> T1) {
- if (!T0)
- return T1;
- if (!T1)
- return T0;
- if (T0 == T1)
- return T0;
- return nullptr;
- }
-
- std::optional<Type *> getPrivatizableType() const override {
- return PrivatizableType;
- }
-
- const std::string getAsStr(Attributor *A) const override {
- return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
- }
-
-protected:
- std::optional<Type *> PrivatizableType;
-};
-
-// TODO: Do this for call site arguments (probably also other values) as well.
-
-struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
- AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
- : AAPrivatizablePtrImpl(IRP, A) {}
-
- /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
- std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
- // If this is a byval argument and we know all the call sites (so we can
- // rewrite them), there is no need to check them explicitly.
- bool UsedAssumedInformation = false;
- SmallVector<Attribute, 1> Attrs;
- A.getAttrs(getIRPosition(), {Attribute::ByVal}, Attrs,
- /* IgnoreSubsumingPositions */ true);
- if (!Attrs.empty() &&
- A.checkForAllCallSites([](AbstractCallSite ACS) { return true; }, *this,
- true, UsedAssumedInformation))
- return Attrs[0].getValueAsType();
-
- std::optional<Type *> Ty;
- unsigned ArgNo = getIRPosition().getCallSiteArgNo();
-
- // Make sure the associated call site argument has the same type at all call
- // sites and it is an allocation we know is safe to privatize, for now that
- // means we only allow alloca instructions.
- // TODO: We can additionally analyze the accesses in the callee to create
- // the type from that information instead. That is a little more
- // involved and will be done in a follow up patch.
- auto CallSiteCheck = [&](AbstractCallSite ACS) {
- IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
- // Check if a coresponding argument was found or if it is one not
- // associated (which can happen for callback calls).
- if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
- return false;
-
- // Check that all call sites agree on a type.
- auto *PrivCSArgAA =
- A.getAAFor<AAPrivatizablePtr>(*this, ACSArgPos, DepClassTy::REQUIRED);
- if (!PrivCSArgAA)
- return false;
- std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
-
- LLVM_DEBUG({
- dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
- if (CSTy && *CSTy)
- (*CSTy)->print(dbgs());
- else if (CSTy)
- dbgs() << "<nullptr>";
- else
- dbgs() << "<none>";
- });
-
- Ty = combineTypes(Ty, CSTy);
-
- LLVM_DEBUG({
- dbgs() << " : New Type: ";
- if (Ty && *Ty)
- (*Ty)->print(dbgs());
- else if (Ty)
- dbgs() << "<nullptr>";
- else
- dbgs() << "<none>";
- dbgs() << "\n";
- });
-
- return !Ty || *Ty;
- };
-
- if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
- UsedAssumedInformation))
- return nullptr;
- return Ty;
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- PrivatizableType = identifyPrivatizableType(A);
- if (!PrivatizableType)
- return ChangeStatus::UNCHANGED;
- if (!*PrivatizableType)
- return indicatePessimisticFixpoint();
-
- // The dependence is optional so we don't give up once we give up on the
- // alignment.
- A.getAAFor<AAAlign>(*this, IRPosition::value(getAssociatedValue()),
- DepClassTy::OPTIONAL);
-
- // Avoid arguments with padding for now.
- if (!A.hasAttr(getIRPosition(), Attribute::ByVal) &&
- !isDenselyPacked(*PrivatizableType, A.getInfoCache().getDL())) {
- LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
- return indicatePessimisticFixpoint();
- }
-
- // Collect the types that will replace the privatizable type in the function
- // signature.
- SmallVector<Type *, 16> ReplacementTypes;
- identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
-
- // Verify callee and caller agree on how the promoted argument would be
- // passed.
- Function &Fn = *getIRPosition().getAnchorScope();
- const auto *TTI =
- A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(Fn);
- if (!TTI) {
- LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
- << Fn.getName() << "\n");
- return indicatePessimisticFixpoint();
- }
-
- auto CallSiteCheck = [&](AbstractCallSite ACS) {
- CallBase *CB = ACS.getInstruction();
- return TTI->areTypesABICompatible(
- CB->getCaller(),
- dyn_cast_if_present<Function>(CB->getCalledOperand()),
- ReplacementTypes);
- };
- bool UsedAssumedInformation = false;
- if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
- UsedAssumedInformation)) {
- LLVM_DEBUG(
- dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
- << Fn.getName() << "\n");
- return indicatePessimisticFixpoint();
- }
-
- // Register a rewrite of the argument.
- Argument *Arg = getAssociatedArgument();
- if (!A.isValidFunctionSignatureRewrite(*Arg, ReplacementTypes)) {
- LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
- return indicatePessimisticFixpoint();
- }
-
- unsigned ArgNo = Arg->getArgNo();
-
- // Helper to check if for the given call site the associated argument is
- // passed to a callback where the privatization would be different.
- auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
- SmallVector<const Use *, 4> CallbackUses;
- AbstractCallSite::getCallbackUses(CB, CallbackUses);
- for (const Use *U : CallbackUses) {
- AbstractCallSite CBACS(U);
- assert(CBACS && CBACS.isCallbackCall());
- for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
- int CBArgNo = CBACS.getCallArgOperandNo(CBArg);
-
- LLVM_DEBUG({
- dbgs()
- << "[AAPrivatizablePtr] Argument " << *Arg
- << "check if can be privatized in the context of its parent ("
- << Arg->getParent()->getName()
- << ")\n[AAPrivatizablePtr] because it is an argument in a "
- "callback ("
- << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
- << ")\n[AAPrivatizablePtr] " << CBArg << " : "
- << CBACS.getCallArgOperand(CBArg) << " vs "
- << CB.getArgOperand(ArgNo) << "\n"
- << "[AAPrivatizablePtr] " << CBArg << " : "
- << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
- });
-
- if (CBArgNo != int(ArgNo))
- continue;
- const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
- *this, IRPosition::argument(CBArg), DepClassTy::REQUIRED);
- if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
- auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
- if (!CBArgPrivTy)
- continue;
- if (*CBArgPrivTy == PrivatizableType)
- continue;
- }
-
- LLVM_DEBUG({
- dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
- << " cannot be privatized in the context of its parent ("
- << Arg->getParent()->getName()
- << ")\n[AAPrivatizablePtr] because it is an argument in a "
- "callback ("
- << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
- << ").\n[AAPrivatizablePtr] for which the argument "
- "privatization is not compatible.\n";
- });
- return false;
- }
- }
- return true;
- };
-
- // Helper to check if for the given call site the associated argument is
- // passed to a direct call where the privatization would be different.
- auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
- CallBase *DC = cast<CallBase>(ACS.getInstruction());
- int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
- assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
- "Expected a direct call operand for callback call operand");
-
- Function *DCCallee =
- dyn_cast_if_present<Function>(DC->getCalledOperand());
- LLVM_DEBUG({
- dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
- << " check if be privatized in the context of its parent ("
- << Arg->getParent()->getName()
- << ")\n[AAPrivatizablePtr] because it is an argument in a "
- "direct call of ("
- << DCArgNo << "@" << DCCallee->getName() << ").\n";
- });
-
- if (unsigned(DCArgNo) < DCCallee->arg_size()) {
- const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
- *this, IRPosition::argument(*DCCallee->getArg(DCArgNo)),
- DepClassTy::REQUIRED);
- if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
- auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
- if (!DCArgPrivTy)
- return true;
- if (*DCArgPrivTy == PrivatizableType)
- return true;
- }
- }
-
- LLVM_DEBUG({
- dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
- << " cannot be privatized in the context of its parent ("
- << Arg->getParent()->getName()
- << ")\n[AAPrivatizablePtr] because it is an argument in a "
- "direct call of ("
- << ACS.getInstruction()->getCalledOperand()->getName()
- << ").\n[AAPrivatizablePtr] for which the argument "
- "privatization is not compatible.\n";
- });
- return false;
- };
-
- // Helper to check if the associated argument is used at the given abstract
- // call site in a way that is incompatible with the privatization assumed
- // here.
- auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
- if (ACS.isDirectCall())
- return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
- if (ACS.isCallbackCall())
- return IsCompatiblePrivArgOfDirectCS(ACS);
- return false;
- };
-
- if (!A.checkForAllCallSites(IsCompatiblePrivArgOfOtherCallSite, *this, true,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- return ChangeStatus::UNCHANGED;
- }
-
- /// Given a type to private \p PrivType, collect the constituates (which are
- /// used) in \p ReplacementTypes.
- static void
- identifyReplacementTypes(Type *PrivType,
- SmallVectorImpl<Type *> &ReplacementTypes) {
- // TODO: For now we expand the privatization type to the fullest which can
- // lead to dead arguments that need to be removed later.
- assert(PrivType && "Expected privatizable type!");
-
- // Traverse the type, extract constituate types on the outermost level.
- if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
- for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
- ReplacementTypes.push_back(PrivStructType->getElementType(u));
- } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
- ReplacementTypes.append(PrivArrayType->getNumElements(),
- PrivArrayType->getElementType());
- } else {
- ReplacementTypes.push_back(PrivType);
- }
- }
-
- /// Initialize \p Base according to the type \p PrivType at position \p IP.
- /// The values needed are taken from the arguments of \p F starting at
- /// position \p ArgNo.
- static void createInitialization(Type *PrivType, Value &Base, Function &F,
- unsigned ArgNo, BasicBlock::iterator IP) {
- assert(PrivType && "Expected privatizable type!");
-
- IRBuilder<NoFolder> IRB(IP->getParent(), IP);
- const DataLayout &DL = F.getParent()->getDataLayout();
-
- // Traverse the type, build GEPs and stores.
- if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
- const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);
- for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
- Value *Ptr =
- constructPointer(&Base, PrivStructLayout->getElementOffset(u), IRB);
- new StoreInst(F.getArg(ArgNo + u), Ptr, IP);
- }
- } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
- Type *PointeeTy = PrivArrayType->getElementType();
- uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);
- for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
- Value *Ptr = constructPointer(&Base, u * PointeeTySize, IRB);
- new StoreInst(F.getArg(ArgNo + u), Ptr, IP);
- }
- } else {
- new StoreInst(F.getArg(ArgNo), &Base, IP);
- }
- }
-
- /// Extract values from \p Base according to the type \p PrivType at the
- /// call position \p ACS. The values are appended to \p ReplacementValues.
- void createReplacementValues(Align Alignment, Type *PrivType,
- AbstractCallSite ACS, Value *Base,
- SmallVectorImpl<Value *> &ReplacementValues) {
- assert(Base && "Expected base value!");
- assert(PrivType && "Expected privatizable type!");
- Instruction *IP = ACS.getInstruction();
-
- IRBuilder<NoFolder> IRB(IP);
- const DataLayout &DL = IP->getModule()->getDataLayout();
-
- // Traverse the type, build GEPs and loads.
- if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
- const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);
- for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
- Type *PointeeTy = PrivStructType->getElementType(u);
- Value *Ptr =
- constructPointer(Base, PrivStructLayout->getElementOffset(u), IRB);
- LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
- L->setAlignment(Alignment);
- ReplacementValues.push_back(L);
- }
- } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
- Type *PointeeTy = PrivArrayType->getElementType();
- uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);
- for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
- Value *Ptr = constructPointer(Base, u * PointeeTySize, IRB);
- LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
- L->setAlignment(Alignment);
- ReplacementValues.push_back(L);
- }
- } else {
- LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
- L->setAlignment(Alignment);
- ReplacementValues.push_back(L);
- }
- }
-
- /// See AbstractAttribute::manifest(...)
- ChangeStatus manifest(Attributor &A) override {
- if (!PrivatizableType)
- return ChangeStatus::UNCHANGED;
- assert(*PrivatizableType && "Expected privatizable type!");
-
- // Collect all tail calls in the function as we cannot allow new allocas to
- // escape into tail recursion.
- // TODO: Be smarter about new allocas escaping into tail calls.
- SmallVector<CallInst *, 16> TailCalls;
- bool UsedAssumedInformation = false;
- if (!A.checkForAllInstructions(
- [&](Instruction &I) {
- CallInst &CI = cast<CallInst>(I);
- if (CI.isTailCall())
- TailCalls.push_back(&CI);
- return true;
- },
- *this, {Instruction::Call}, UsedAssumedInformation))
- return ChangeStatus::UNCHANGED;
-
- Argument *Arg = getAssociatedArgument();
- // Query AAAlign attribute for alignment of associated argument to
- // determine the best alignment of loads.
- const auto *AlignAA =
- A.getAAFor<AAAlign>(*this, IRPosition::value(*Arg), DepClassTy::NONE);
-
- // Callback to repair the associated function. A new alloca is placed at the
- // beginning and initialized with the values passed through arguments. The
- // new alloca replaces the use of the old pointer argument.
- Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB =
- [=](const Attributor::ArgumentReplacementInfo &ARI,
- Function &ReplacementFn, Function::arg_iterator ArgIt) {
- BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
- BasicBlock::iterator IP = EntryBB.getFirstInsertionPt();
- const DataLayout &DL = IP->getModule()->getDataLayout();
- unsigned AS = DL.getAllocaAddrSpace();
- Instruction *AI = new AllocaInst(*PrivatizableType, AS,
- Arg->getName() + ".priv", IP);
- createInitialization(*PrivatizableType, *AI, ReplacementFn,
- ArgIt->getArgNo(), IP);
-
- if (AI->getType() != Arg->getType())
- AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
- AI, Arg->getType(), "", IP);
- Arg->replaceAllUsesWith(AI);
-
- for (CallInst *CI : TailCalls)
- CI->setTailCall(false);
- };
-
- // Callback to repair a call site of the associated function. The elements
- // of the privatizable type are loaded prior to the call and passed to the
- // new function version.
- Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB =
- [=](const Attributor::ArgumentReplacementInfo &ARI,
- AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
- // When no alignment is specified for the load instruction,
- // natural alignment is assumed.
- createReplacementValues(
- AlignAA ? AlignAA->getAssumedAlign() : Align(0),
- *PrivatizableType, ACS,
- ACS.getCallArgOperand(ARI.getReplacedArg().getArgNo()),
- NewArgOperands);
- };
-
- // Collect the types that will replace the privatizable type in the function
- // signature.
- SmallVector<Type *, 16> ReplacementTypes;
- identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
-
- // Register a rewrite of the argument.
- if (A.registerFunctionSignatureRewrite(*Arg, ReplacementTypes,
- std::move(FnRepairCB),
- std::move(ACSRepairCB)))
- return ChangeStatus::CHANGED;
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
- }
-};
-
-struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
- AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
- : AAPrivatizablePtrImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- // TODO: We can privatize more than arguments.
- indicatePessimisticFixpoint();
- }
-
- ChangeStatus updateImpl(Attributor &A) override {
- llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
- "updateImpl will not be called");
- }
-
- /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
- std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
- Value *Obj = getUnderlyingObject(&getAssociatedValue());
- if (!Obj) {
- LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
- return nullptr;
- }
-
- if (auto *AI = dyn_cast<AllocaInst>(Obj))
- if (auto *CI = dyn_cast<ConstantInt>(AI->getArraySize()))
- if (CI->isOne())
- return AI->getAllocatedType();
- if (auto *Arg = dyn_cast<Argument>(Obj)) {
- auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
- *this, IRPosition::argument(*Arg), DepClassTy::REQUIRED);
- if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
- return PrivArgAA->getPrivatizableType();
- }
-
- LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
- "alloca nor privatizable argument: "
- << *Obj << "!\n");
- return nullptr;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
- }
-};
-
-struct AAPrivatizablePtrCallSiteArgument final
- : public AAPrivatizablePtrFloating {
- AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAPrivatizablePtrFloating(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- if (A.hasAttr(getIRPosition(), Attribute::ByVal))
- indicateOptimisticFixpoint();
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- PrivatizableType = identifyPrivatizableType(A);
- if (!PrivatizableType)
- return ChangeStatus::UNCHANGED;
- if (!*PrivatizableType)
- return indicatePessimisticFixpoint();
-
- const IRPosition &IRP = getIRPosition();
- bool IsKnownNoCapture;
- bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, IRP, DepClassTy::REQUIRED, IsKnownNoCapture);
- if (!IsAssumedNoCapture) {
- LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
- return indicatePessimisticFixpoint();
- }
-
- bool IsKnownNoAlias;
- if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
- A, this, IRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
- LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
- return indicatePessimisticFixpoint();
- }
-
- bool IsKnown;
- if (!AA::isAssumedReadOnly(A, IRP, *this, IsKnown)) {
- LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
- return indicatePessimisticFixpoint();
- }
-
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
- }
-};
-
-struct AAPrivatizablePtrCallSiteReturned final
- : public AAPrivatizablePtrFloating {
- AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AAPrivatizablePtrFloating(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- // TODO: We can privatize more than arguments.
- indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
- }
-};
-
-struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
- AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
- : AAPrivatizablePtrFloating(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- // TODO: We can privatize more than arguments.
- indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
- }
-};
-} // namespace
-
-/// -------------------- Memory Behavior Attributes ----------------------------
-/// Includes read-none, read-only, and write-only.
-/// ----------------------------------------------------------------------------
-namespace {
-struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
- AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
- : AAMemoryBehavior(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- intersectAssumedBits(BEST_STATE);
- getKnownStateFromValue(A, getIRPosition(), getState());
- AAMemoryBehavior::initialize(A);
- }
-
- /// Return the memory behavior information encoded in the IR for \p IRP.
- static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
- BitIntegerState &State,
- bool IgnoreSubsumingPositions = false) {
- SmallVector<Attribute, 2> Attrs;
- A.getAttrs(IRP, AttrKinds, Attrs, IgnoreSubsumingPositions);
- for (const Attribute &Attr : Attrs) {
- switch (Attr.getKindAsEnum()) {
- case Attribute::ReadNone:
- State.addKnownBits(NO_ACCESSES);
- break;
- case Attribute::ReadOnly:
- State.addKnownBits(NO_WRITES);
- break;
- case Attribute::WriteOnly:
- State.addKnownBits(NO_READS);
- break;
- default:
- llvm_unreachable("Unexpected attribute!");
- }
- }
-
- if (auto *I = dyn_cast<Instruction>(&IRP.getAnchorValue())) {
- if (!I->mayReadFromMemory())
- State.addKnownBits(NO_READS);
- if (!I->mayWriteToMemory())
- State.addKnownBits(NO_WRITES);
- }
- }
-
- /// See AbstractAttribute::getDeducedAttributes(...).
- void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
- SmallVectorImpl<Attribute> &Attrs) const override {
- assert(Attrs.size() == 0);
- if (isAssumedReadNone())
- Attrs.push_back(Attribute::get(Ctx, Attribute::ReadNone));
- else if (isAssumedReadOnly())
- Attrs.push_back(Attribute::get(Ctx, Attribute::ReadOnly));
- else if (isAssumedWriteOnly())
- Attrs.push_back(Attribute::get(Ctx, Attribute::WriteOnly));
- assert(Attrs.size() <= 1);
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- const IRPosition &IRP = getIRPosition();
-
- if (A.hasAttr(IRP, Attribute::ReadNone,
- /* IgnoreSubsumingPositions */ true))
- return ChangeStatus::UNCHANGED;
-
- // Check if we would improve the existing attributes first.
- SmallVector<Attribute, 4> DeducedAttrs;
- getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);
- if (llvm::all_of(DeducedAttrs, [&](const Attribute &Attr) {
- return A.hasAttr(IRP, Attr.getKindAsEnum(),
- /* IgnoreSubsumingPositions */ true);
- }))
- return ChangeStatus::UNCHANGED;
-
- // Clear existing attributes.
- A.removeAttrs(IRP, AttrKinds);
- // Clear conflicting writable attribute.
- if (isAssumedReadOnly())
- A.removeAttrs(IRP, Attribute::Writable);
-
- // Use the generic manifest method.
- return IRAttribute::manifest(A);
- }
-
- /// See AbstractState::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- if (isAssumedReadNone())
- return "readnone";
- if (isAssumedReadOnly())
- return "readonly";
- if (isAssumedWriteOnly())
- return "writeonly";
- return "may-read/write";
- }
-
- /// The set of IR attributes AAMemoryBehavior deals with.
- static const Attribute::AttrKind AttrKinds[3];
-};
-
-const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
- Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
-
-/// Memory behavior attribute for a floating value.
-struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
- AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
- : AAMemoryBehaviorImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override;
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- if (isAssumedReadNone())
- STATS_DECLTRACK_FLOATING_ATTR(readnone)
- else if (isAssumedReadOnly())
- STATS_DECLTRACK_FLOATING_ATTR(readonly)
- else if (isAssumedWriteOnly())
- STATS_DECLTRACK_FLOATING_ATTR(writeonly)
- }
-
-private:
- /// Return true if users of \p UserI might access the underlying
- /// variable/location described by \p U and should therefore be analyzed.
- bool followUsersOfUseIn(Attributor &A, const Use &U,
- const Instruction *UserI);
-
- /// Update the state according to the effect of use \p U in \p UserI.
- void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
-};
-
-/// Memory behavior attribute for function argument.
-struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
- AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
- : AAMemoryBehaviorFloating(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- intersectAssumedBits(BEST_STATE);
- const IRPosition &IRP = getIRPosition();
- // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
- // can query it when we use has/getAttr. That would allow us to reuse the
- // initialize of the base class here.
- bool HasByVal = A.hasAttr(IRP, {Attribute::ByVal},
- /* IgnoreSubsumingPositions */ true);
- getKnownStateFromValue(A, IRP, getState(),
- /* IgnoreSubsumingPositions */ HasByVal);
- }
-
- ChangeStatus manifest(Attributor &A) override {
- // TODO: Pointer arguments are not supported on vectors of pointers yet.
- if (!getAssociatedValue().getType()->isPointerTy())
- return ChangeStatus::UNCHANGED;
-
- // TODO: From readattrs.ll: "inalloca parameters are always
- // considered written"
- if (A.hasAttr(getIRPosition(),
- {Attribute::InAlloca, Attribute::Preallocated})) {
- removeKnownBits(NO_WRITES);
- removeAssumedBits(NO_WRITES);
- }
- A.removeAttrs(getIRPosition(), AttrKinds);
- return AAMemoryBehaviorFloating::manifest(A);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- if (isAssumedReadNone())
- STATS_DECLTRACK_ARG_ATTR(readnone)
- else if (isAssumedReadOnly())
- STATS_DECLTRACK_ARG_ATTR(readonly)
- else if (isAssumedWriteOnly())
- STATS_DECLTRACK_ARG_ATTR(writeonly)
- }
-};
-
-struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
- AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAMemoryBehaviorArgument(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- // If we don't have an associated attribute this is either a variadic call
- // or an indirect call, either way, nothing to do here.
- Argument *Arg = getAssociatedArgument();
- if (!Arg) {
- indicatePessimisticFixpoint();
- return;
- }
- if (Arg->hasByValAttr()) {
- addKnownBits(NO_WRITES);
- removeKnownBits(NO_READS);
- removeAssumedBits(NO_READS);
- }
- AAMemoryBehaviorArgument::initialize(A);
- if (getAssociatedFunction()->isDeclaration())
- indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // TODO: Once we have call site specific value information we can provide
- // call site specific liveness liveness information and then it makes
- // sense to specialize attributes for call sites arguments instead of
- // redirecting requests to the callee argument.
- Argument *Arg = getAssociatedArgument();
- const IRPosition &ArgPos = IRPosition::argument(*Arg);
- auto *ArgAA =
- A.getAAFor<AAMemoryBehavior>(*this, ArgPos, DepClassTy::REQUIRED);
- if (!ArgAA)
- return indicatePessimisticFixpoint();
- return clampStateAndIndicateChange(getState(), ArgAA->getState());
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- if (isAssumedReadNone())
- STATS_DECLTRACK_CSARG_ATTR(readnone)
- else if (isAssumedReadOnly())
- STATS_DECLTRACK_CSARG_ATTR(readonly)
- else if (isAssumedWriteOnly())
- STATS_DECLTRACK_CSARG_ATTR(writeonly)
- }
-};
-
-/// Memory behavior attribute for a call site return position.
-struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
- AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AAMemoryBehaviorFloating(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- AAMemoryBehaviorImpl::initialize(A);
- }
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- // We do not annotate returned values.
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {}
-};
-
-/// An AA to represent the memory behavior function attributes.
-struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
- AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
- : AAMemoryBehaviorImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(Attributor &A).
- ChangeStatus updateImpl(Attributor &A) override;
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- // TODO: It would be better to merge this with AAMemoryLocation, so that
- // we could determine read/write per location. This would also have the
- // benefit of only one place trying to manifest the memory attribute.
- Function &F = cast<Function>(getAnchorValue());
- MemoryEffects ME = MemoryEffects::unknown();
- if (isAssumedReadNone())
- ME = MemoryEffects::none();
- else if (isAssumedReadOnly())
- ME = MemoryEffects::readOnly();
- else if (isAssumedWriteOnly())
- ME = MemoryEffects::writeOnly();
-
- A.removeAttrs(getIRPosition(), AttrKinds);
- // Clear conflicting writable attribute.
- if (ME.onlyReadsMemory())
- for (Argument &Arg : F.args())
- A.removeAttrs(IRPosition::argument(Arg), Attribute::Writable);
- return A.manifestAttrs(getIRPosition(),
- Attribute::getWithMemoryEffects(F.getContext(), ME));
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- if (isAssumedReadNone())
- STATS_DECLTRACK_FN_ATTR(readnone)
- else if (isAssumedReadOnly())
- STATS_DECLTRACK_FN_ATTR(readonly)
- else if (isAssumedWriteOnly())
- STATS_DECLTRACK_FN_ATTR(writeonly)
- }
-};
-
-/// AAMemoryBehavior attribute for call sites.
-struct AAMemoryBehaviorCallSite final
- : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
- AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- // TODO: Deduplicate this with AAMemoryBehaviorFunction.
- CallBase &CB = cast<CallBase>(getAnchorValue());
- MemoryEffects ME = MemoryEffects::unknown();
- if (isAssumedReadNone())
- ME = MemoryEffects::none();
- else if (isAssumedReadOnly())
- ME = MemoryEffects::readOnly();
- else if (isAssumedWriteOnly())
- ME = MemoryEffects::writeOnly();
-
- A.removeAttrs(getIRPosition(), AttrKinds);
- // Clear conflicting writable attribute.
- if (ME.onlyReadsMemory())
- for (Use &U : CB.args())
- A.removeAttrs(IRPosition::callsite_argument(CB, U.getOperandNo()),
- Attribute::Writable);
- return A.manifestAttrs(
- getIRPosition(), Attribute::getWithMemoryEffects(CB.getContext(), ME));
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- if (isAssumedReadNone())
- STATS_DECLTRACK_CS_ATTR(readnone)
- else if (isAssumedReadOnly())
- STATS_DECLTRACK_CS_ATTR(readonly)
- else if (isAssumedWriteOnly())
- STATS_DECLTRACK_CS_ATTR(writeonly)
- }
-};
-
-ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
-
- // The current assumed state used to determine a change.
- auto AssumedState = getAssumed();
-
- auto CheckRWInst = [&](Instruction &I) {
- // If the instruction has an own memory behavior state, use it to restrict
- // the local state. No further analysis is required as the other memory
- // state is as optimistic as it gets.
- if (const auto *CB = dyn_cast<CallBase>(&I)) {
- const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
- *this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED);
- if (MemBehaviorAA) {
- intersectAssumedBits(MemBehaviorAA->getAssumed());
- return !isAtFixpoint();
- }
- }
-
- // Remove access kind modifiers if necessary.
- if (I.mayReadFromMemory())
- removeAssumedBits(NO_READS);
- if (I.mayWriteToMemory())
- removeAssumedBits(NO_WRITES);
- return !isAtFixpoint();
- };
-
- bool UsedAssumedInformation = false;
- if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
- : ChangeStatus::UNCHANGED;
-}
-
-ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
-
- const IRPosition &IRP = getIRPosition();
- const IRPosition &FnPos = IRPosition::function_scope(IRP);
- AAMemoryBehavior::StateType &S = getState();
-
- // First, check the function scope. We take the known information and we avoid
- // work if the assumed information implies the current assumed information for
- // this attribute. This is a valid for all but byval arguments.
- Argument *Arg = IRP.getAssociatedArgument();
- AAMemoryBehavior::base_t FnMemAssumedState =
- AAMemoryBehavior::StateType::getWorstState();
- if (!Arg || !Arg->hasByValAttr()) {
- const auto *FnMemAA =
- A.getAAFor<AAMemoryBehavior>(*this, FnPos, DepClassTy::OPTIONAL);
- if (FnMemAA) {
- FnMemAssumedState = FnMemAA->getAssumed();
- S.addKnownBits(FnMemAA->getKnown());
- if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
- return ChangeStatus::UNCHANGED;
- }
- }
-
- // The current assumed state used to determine a change.
- auto AssumedState = S.getAssumed();
-
- // Make sure the value is not captured (except through "return"), if
- // it is, any information derived would be irrelevant anyway as we cannot
- // check the potential aliases introduced by the capture. However, no need
- // to fall back to anythign less optimistic than the function state.
- bool IsKnownNoCapture;
- const AANoCapture *ArgNoCaptureAA = nullptr;
- bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture, false,
- &ArgNoCaptureAA);
-
- if (!IsAssumedNoCapture &&
- (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
- S.intersectAssumedBits(FnMemAssumedState);
- return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
- : ChangeStatus::UNCHANGED;
- }
-
- // Visit and expand uses until all are analyzed or a fixpoint is reached.
- auto UsePred = [&](const Use &U, bool &Follow) -> bool {
- Instruction *UserI = cast<Instruction>(U.getUser());
- LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
- << " \n");
-
- // Droppable users, e.g., llvm::assume does not actually perform any action.
- if (UserI->isDroppable())
- return true;
-
- // Check if the users of UserI should also be visited.
- Follow = followUsersOfUseIn(A, U, UserI);
-
- // If UserI might touch memory we analyze the use in detail.
- if (UserI->mayReadOrWriteMemory())
- analyzeUseIn(A, U, UserI);
-
- return !isAtFixpoint();
- };
-
- if (!A.checkForAllUses(UsePred, *this, getAssociatedValue()))
- return indicatePessimisticFixpoint();
-
- return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
- : ChangeStatus::UNCHANGED;
-}
-
-bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
- const Instruction *UserI) {
- // The loaded value is unrelated to the pointer argument, no need to
- // follow the users of the load.
- if (isa<LoadInst>(UserI) || isa<ReturnInst>(UserI))
- return false;
-
- // By default we follow all uses assuming UserI might leak information on U,
- // we have special handling for call sites operands though.
- const auto *CB = dyn_cast<CallBase>(UserI);
- if (!CB || !CB->isArgOperand(&U))
- return true;
-
- // If the use is a call argument known not to be captured, the users of
- // the call do not need to be visited because they have to be unrelated to
- // the input. Note that this check is not trivial even though we disallow
- // general capturing of the underlying argument. The reason is that the
- // call might the argument "through return", which we allow and for which we
- // need to check call users.
- if (U.get()->getType()->isPointerTy()) {
- unsigned ArgNo = CB->getArgOperandNo(&U);
- bool IsKnownNoCapture;
- return !AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, IRPosition::callsite_argument(*CB, ArgNo),
- DepClassTy::OPTIONAL, IsKnownNoCapture);
- }
-
- return true;
-}
-
-void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
- const Instruction *UserI) {
- assert(UserI->mayReadOrWriteMemory());
-
- switch (UserI->getOpcode()) {
- default:
- // TODO: Handle all atomics and other side-effect operations we know of.
- break;
- case Instruction::Load:
- // Loads cause the NO_READS property to disappear.
- removeAssumedBits(NO_READS);
- return;
-
- case Instruction::Store:
- // Stores cause the NO_WRITES property to disappear if the use is the
- // pointer operand. Note that while capturing was taken care of somewhere
- // else we need to deal with stores of the value that is not looked through.
- if (cast<StoreInst>(UserI)->getPointerOperand() == U.get())
- removeAssumedBits(NO_WRITES);
- else
- indicatePessimisticFixpoint();
- return;
-
- case Instruction::Call:
- case Instruction::CallBr:
- case Instruction::Invoke: {
- // For call sites we look at the argument memory behavior attribute (this
- // could be recursive!) in order to restrict our own state.
- const auto *CB = cast<CallBase>(UserI);
-
- // Give up on operand bundles.
- if (CB->isBundleOperand(&U)) {
- indicatePessimisticFixpoint();
- return;
- }
-
- // Calling a function does read the function pointer, maybe write it if the
- // function is self-modifying.
- if (CB->isCallee(&U)) {
- removeAssumedBits(NO_READS);
- break;
- }
-
- // Adjust the possible access behavior based on the information on the
- // argument.
- IRPosition Pos;
- if (U.get()->getType()->isPointerTy())
- Pos = IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U));
- else
- Pos = IRPosition::callsite_function(*CB);
- const auto *MemBehaviorAA =
- A.getAAFor<AAMemoryBehavior>(*this, Pos, DepClassTy::OPTIONAL);
- if (!MemBehaviorAA)
- break;
- // "assumed" has at most the same bits as the MemBehaviorAA assumed
- // and at least "known".
- intersectAssumedBits(MemBehaviorAA->getAssumed());
- return;
- }
- };
-
- // Generally, look at the "may-properties" and adjust the assumed state if we
- // did not trigger special handling before.
- if (UserI->mayReadFromMemory())
- removeAssumedBits(NO_READS);
- if (UserI->mayWriteToMemory())
- removeAssumedBits(NO_WRITES);
-}
-} // namespace
-
-/// -------------------- Memory Locations Attributes ---------------------------
-/// Includes read-none, argmemonly, inaccessiblememonly,
-/// inaccessiblememorargmemonly
-/// ----------------------------------------------------------------------------
-
-std::string AAMemoryLocation::getMemoryLocationsAsStr(
- AAMemoryLocation::MemoryLocationsKind MLK) {
- if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
- return "all memory";
- if (MLK == AAMemoryLocation::NO_LOCATIONS)
- return "no memory";
- std::string S = "memory:";
- if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
- S += "stack,";
- if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
- S += "constant,";
- if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM))
- S += "internal global,";
- if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM))
- S += "external global,";
- if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
- S += "argument,";
- if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM))
- S += "inaccessible,";
- if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
- S += "malloced,";
- if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
- S += "unknown,";
- S.pop_back();
- return S;
-}
-
-namespace {
-struct AAMemoryLocationImpl : public AAMemoryLocation {
-
- AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
- : AAMemoryLocation(IRP, A), Allocator(A.Allocator) {
- AccessKind2Accesses.fill(nullptr);
- }
-
- ~AAMemoryLocationImpl() {
- // The AccessSets are allocated via a BumpPtrAllocator, we call
- // the destructor manually.
- for (AccessSet *AS : AccessKind2Accesses)
- if (AS)
- AS->~AccessSet();
- }
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- intersectAssumedBits(BEST_STATE);
- getKnownStateFromValue(A, getIRPosition(), getState());
- AAMemoryLocation::initialize(A);
- }
-
- /// Return the memory behavior information encoded in the IR for \p IRP.
- static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
- BitIntegerState &State,
- bool IgnoreSubsumingPositions = false) {
- // For internal functions we ignore `argmemonly` and
- // `inaccessiblememorargmemonly` as we might break it via interprocedural
- // constant propagation. It is unclear if this is the best way but it is
- // unlikely this will cause real performance problems. If we are deriving
- // attributes for the anchor function we even remove the attribute in
- // addition to ignoring it.
- // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
- // MemoryEffects::Other as a possible location.
- bool UseArgMemOnly = true;
- Function *AnchorFn = IRP.getAnchorScope();
- if (AnchorFn && A.isRunOn(*AnchorFn))
- UseArgMemOnly = !AnchorFn->hasLocalLinkage();
-
- SmallVector<Attribute, 2> Attrs;
- A.getAttrs(IRP, {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
- for (const Attribute &Attr : Attrs) {
- // TODO: We can map MemoryEffects to Attributor locations more precisely.
- MemoryEffects ME = Attr.getMemoryEffects();
- if (ME.doesNotAccessMemory()) {
- State.addKnownBits(NO_LOCAL_MEM | NO_CONST_MEM);
- continue;
- }
- if (ME.onlyAccessesInaccessibleMem()) {
- State.addKnownBits(inverseLocation(NO_INACCESSIBLE_MEM, true, true));
- continue;
- }
- if (ME.onlyAccessesArgPointees()) {
- if (UseArgMemOnly)
- State.addKnownBits(inverseLocation(NO_ARGUMENT_MEM, true, true));
- else {
- // Remove location information, only keep read/write info.
- ME = MemoryEffects(ME.getModRef());
- A.manifestAttrs(IRP,
- Attribute::getWithMemoryEffects(
- IRP.getAnchorValue().getContext(), ME),
- /*ForceReplace*/ true);
- }
- continue;
- }
- if (ME.onlyAccessesInaccessibleOrArgMem()) {
- if (UseArgMemOnly)
- State.addKnownBits(inverseLocation(
- NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, true, true));
- else {
- // Remove location information, only keep read/write info.
- ME = MemoryEffects(ME.getModRef());
- A.manifestAttrs(IRP,
- Attribute::getWithMemoryEffects(
- IRP.getAnchorValue().getContext(), ME),
- /*ForceReplace*/ true);
- }
- continue;
- }
- }
- }
-
- /// See AbstractAttribute::getDeducedAttributes(...).
- void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
- SmallVectorImpl<Attribute> &Attrs) const override {
- // TODO: We can map Attributor locations to MemoryEffects more precisely.
- assert(Attrs.size() == 0);
- if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
- if (isAssumedReadNone())
- Attrs.push_back(
- Attribute::getWithMemoryEffects(Ctx, MemoryEffects::none()));
- else if (isAssumedInaccessibleMemOnly())
- Attrs.push_back(Attribute::getWithMemoryEffects(
- Ctx, MemoryEffects::inaccessibleMemOnly()));
- else if (isAssumedArgMemOnly())
- Attrs.push_back(
- Attribute::getWithMemoryEffects(Ctx, MemoryEffects::argMemOnly()));
- else if (isAssumedInaccessibleOrArgMemOnly())
- Attrs.push_back(Attribute::getWithMemoryEffects(
- Ctx, MemoryEffects::inaccessibleOrArgMemOnly()));
- }
- assert(Attrs.size() <= 1);
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
- // provide per-location modref information here.
- const IRPosition &IRP = getIRPosition();
-
- SmallVector<Attribute, 1> DeducedAttrs;
- getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);
- if (DeducedAttrs.size() != 1)
- return ChangeStatus::UNCHANGED;
- MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
-
- return A.manifestAttrs(IRP, Attribute::getWithMemoryEffects(
- IRP.getAnchorValue().getContext(), ME));
- }
-
- /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).
- bool checkForAllAccessesToMemoryKind(
- function_ref<bool(const Instruction *, const Value *, AccessKind,
- MemoryLocationsKind)>
- Pred,
- MemoryLocationsKind RequestedMLK) const override {
- if (!isValidState())
- return false;
-
- MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
- if (AssumedMLK == NO_LOCATIONS)
- return true;
-
- unsigned Idx = 0;
- for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
- CurMLK *= 2, ++Idx) {
- if (CurMLK & RequestedMLK)
- continue;
-
- if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
- for (const AccessInfo &AI : *Accesses)
- if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
- return false;
- }
-
- return true;
- }
-
- ChangeStatus indicatePessimisticFixpoint() override {
- // If we give up and indicate a pessimistic fixpoint this instruction will
- // become an access for all potential access kinds:
- // TODO: Add pointers for argmemonly and globals to improve the results of
- // checkForAllAccessesToMemoryKind.
- bool Changed = false;
- MemoryLocationsKind KnownMLK = getKnown();
- Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
- for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
- if (!(CurMLK & KnownMLK))
- updateStateAndAccessesMap(getState(), CurMLK, I, nullptr, Changed,
- getAccessKindFromInst(I));
- return AAMemoryLocation::indicatePessimisticFixpoint();
- }
-
-protected:
- /// Helper struct to tie together an instruction that has a read or write
- /// effect with the pointer it accesses (if any).
- struct AccessInfo {
-
- /// The instruction that caused the access.
- const Instruction *I;
-
- /// The base pointer that is accessed, or null if unknown.
- const Value *Ptr;
-
- /// The kind of access (read/write/read+write).
- AccessKind Kind;
-
- bool operator==(const AccessInfo &RHS) const {
- return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
- }
- bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
- if (LHS.I != RHS.I)
- return LHS.I < RHS.I;
- if (LHS.Ptr != RHS.Ptr)
- return LHS.Ptr < RHS.Ptr;
- if (LHS.Kind != RHS.Kind)
- return LHS.Kind < RHS.Kind;
- return false;
- }
- };
-
- /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
- /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
- using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
- std::array<AccessSet *, llvm::CTLog2<VALID_STATE>()> AccessKind2Accesses;
-
- /// Categorize the pointer arguments of CB that might access memory in
- /// AccessedLoc and update the state and access map accordingly.
- void
- categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
- AAMemoryLocation::StateType &AccessedLocs,
- bool &Changed);
-
- /// Return the kind(s) of location that may be accessed by \p V.
- AAMemoryLocation::MemoryLocationsKind
- categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
-
- /// Return the access kind as determined by \p I.
- AccessKind getAccessKindFromInst(const Instruction *I) {
- AccessKind AK = READ_WRITE;
- if (I) {
- AK = I->mayReadFromMemory() ? READ : NONE;
- AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
- }
- return AK;
- }
-
- /// Update the state \p State and the AccessKind2Accesses given that \p I is
- /// an access of kind \p AK to a \p MLK memory location with the access
- /// pointer \p Ptr.
- void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
- MemoryLocationsKind MLK, const Instruction *I,
- const Value *Ptr, bool &Changed,
- AccessKind AK = READ_WRITE) {
-
- assert(isPowerOf2_32(MLK) && "Expected a single location set!");
- auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(MLK)];
- if (!Accesses)
- Accesses = new (Allocator) AccessSet();
- Changed |= Accesses->insert(AccessInfo{I, Ptr, AK}).second;
- if (MLK == NO_UNKOWN_MEM)
- MLK = NO_LOCATIONS;
- State.removeAssumedBits(MLK);
- }
-
- /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
- /// arguments, and update the state and access map accordingly.
- void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
- AAMemoryLocation::StateType &State, bool &Changed,
- unsigned AccessAS = 0);
-
- /// Used to allocate access sets.
- BumpPtrAllocator &Allocator;
-};
-
-void AAMemoryLocationImpl::categorizePtrValue(
- Attributor &A, const Instruction &I, const Value &Ptr,
- AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
- LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
- << Ptr << " ["
- << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
-
- auto Pred = [&](Value &Obj) {
- unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
- // TODO: recognize the TBAA used for constant accesses.
- MemoryLocationsKind MLK = NO_LOCATIONS;
-
- // Filter accesses to constant (GPU) memory if we have an AS at the access
- // site or the object is known to actually have the associated AS.
- if ((AccessAS == (unsigned)AA::GPUAddressSpace::Constant ||
- (ObjectAS == (unsigned)AA::GPUAddressSpace::Constant &&
- isIdentifiedObject(&Obj))) &&
- AA::isGPU(*I.getModule()))
- return true;
-
- if (isa<UndefValue>(&Obj))
- return true;
- if (isa<Argument>(&Obj)) {
- // TODO: For now we do not treat byval arguments as local copies performed
- // on the call edge, though, we should. To make that happen we need to
- // teach various passes, e.g., DSE, about the copy effect of a byval. That
- // would also allow us to mark functions only accessing byval arguments as
- // readnone again, arguably their accesses have no effect outside of the
- // function, like accesses to allocas.
- MLK = NO_ARGUMENT_MEM;
- } else if (auto *GV = dyn_cast<GlobalValue>(&Obj)) {
- // Reading constant memory is not treated as a read "effect" by the
- // function attr pass so we won't neither. Constants defined by TBAA are
- // similar. (We know we do not write it because it is constant.)
- if (auto *GVar = dyn_cast<GlobalVariable>(GV))
- if (GVar->isConstant())
- return true;
-
- if (GV->hasLocalLinkage())
- MLK = NO_GLOBAL_INTERNAL_MEM;
- else
- MLK = NO_GLOBAL_EXTERNAL_MEM;
- } else if (isa<ConstantPointerNull>(&Obj) &&
- (!NullPointerIsDefined(getAssociatedFunction(), AccessAS) ||
- !NullPointerIsDefined(getAssociatedFunction(), ObjectAS))) {
- return true;
- } else if (isa<AllocaInst>(&Obj)) {
- MLK = NO_LOCAL_MEM;
- } else if (const auto *CB = dyn_cast<CallBase>(&Obj)) {
- bool IsKnownNoAlias;
- if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
- A, this, IRPosition::callsite_returned(*CB), DepClassTy::OPTIONAL,
- IsKnownNoAlias))
- MLK = NO_MALLOCED_MEM;
- else
- MLK = NO_UNKOWN_MEM;
- } else {
- MLK = NO_UNKOWN_MEM;
- }
-
- assert(MLK != NO_LOCATIONS && "No location specified!");
- LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
- << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
- updateStateAndAccessesMap(State, MLK, &I, &Obj, Changed,
- getAccessKindFromInst(&I));
-
- return true;
- };
-
- const auto *AA = A.getAAFor<AAUnderlyingObjects>(
- *this, IRPosition::value(Ptr), DepClassTy::OPTIONAL);
- if (!AA || !AA->forallUnderlyingObjects(Pred, AA::Intraprocedural)) {
- LLVM_DEBUG(
- dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
- updateStateAndAccessesMap(State, NO_UNKOWN_MEM, &I, nullptr, Changed,
- getAccessKindFromInst(&I));
- return;
- }
-
- LLVM_DEBUG(
- dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
- << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
-}
-
-void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
- Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
- bool &Changed) {
- for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
-
- // Skip non-pointer arguments.
- const Value *ArgOp = CB.getArgOperand(ArgNo);
- if (!ArgOp->getType()->isPtrOrPtrVectorTy())
- continue;
-
- // Skip readnone arguments.
- const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
- const auto *ArgOpMemLocationAA =
- A.getAAFor<AAMemoryBehavior>(*this, ArgOpIRP, DepClassTy::OPTIONAL);
-
- if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
- continue;
-
- // Categorize potentially accessed pointer arguments as if there was an
- // access instruction with them as pointer.
- categorizePtrValue(A, CB, *ArgOp, AccessedLocs, Changed);
- }
-}
-
-AAMemoryLocation::MemoryLocationsKind
-AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
- bool &Changed) {
- LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
- << I << "\n");
-
- AAMemoryLocation::StateType AccessedLocs;
- AccessedLocs.intersectAssumedBits(NO_LOCATIONS);
-
- if (auto *CB = dyn_cast<CallBase>(&I)) {
-
- // First check if we assume any memory is access is visible.
- const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
- *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);
- LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
- << " [" << CBMemLocationAA << "]\n");
- if (!CBMemLocationAA) {
- updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr,
- Changed, getAccessKindFromInst(&I));
- return NO_UNKOWN_MEM;
- }
-
- if (CBMemLocationAA->isAssumedReadNone())
- return NO_LOCATIONS;
-
- if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
- updateStateAndAccessesMap(AccessedLocs, NO_INACCESSIBLE_MEM, &I, nullptr,
- Changed, getAccessKindFromInst(&I));
- return AccessedLocs.getAssumed();
- }
-
- uint32_t CBAssumedNotAccessedLocs =
- CBMemLocationAA->getAssumedNotAccessedLocation();
-
- // Set the argmemonly and global bit as we handle them separately below.
- uint32_t CBAssumedNotAccessedLocsNoArgMem =
- CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
-
- for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
- if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
- continue;
- updateStateAndAccessesMap(AccessedLocs, CurMLK, &I, nullptr, Changed,
- getAccessKindFromInst(&I));
- }
-
- // Now handle global memory if it might be accessed. This is slightly tricky
- // as NO_GLOBAL_MEM has multiple bits set.
- bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
- if (HasGlobalAccesses) {
- auto AccessPred = [&](const Instruction *, const Value *Ptr,
- AccessKind Kind, MemoryLocationsKind MLK) {
- updateStateAndAccessesMap(AccessedLocs, MLK, &I, Ptr, Changed,
- getAccessKindFromInst(&I));
- return true;
- };
- if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
- AccessPred, inverseLocation(NO_GLOBAL_MEM, false, false)))
- return AccessedLocs.getWorstState();
- }
-
- LLVM_DEBUG(
- dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
- << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
-
- // Now handle argument memory if it might be accessed.
- bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
- if (HasArgAccesses)
- categorizeArgumentPointerLocations(A, *CB, AccessedLocs, Changed);
-
- LLVM_DEBUG(
- dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
- << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
-
- return AccessedLocs.getAssumed();
- }
-
- if (const Value *Ptr = getPointerOperand(&I, /* AllowVolatile */ true)) {
- LLVM_DEBUG(
- dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
- << I << " [" << *Ptr << "]\n");
- categorizePtrValue(A, I, *Ptr, AccessedLocs, Changed,
- Ptr->getType()->getPointerAddressSpace());
- return AccessedLocs.getAssumed();
- }
-
- LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
- << I << "\n");
- updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr, Changed,
- getAccessKindFromInst(&I));
- return AccessedLocs.getAssumed();
-}
-
-/// An AA to represent the memory behavior function attributes.
-struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
- AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
- : AAMemoryLocationImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(Attributor &A).
- ChangeStatus updateImpl(Attributor &A) override {
-
- const auto *MemBehaviorAA =
- A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
- if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
- if (MemBehaviorAA->isKnownReadNone())
- return indicateOptimisticFixpoint();
- assert(isAssumedReadNone() &&
- "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
- A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);
- return ChangeStatus::UNCHANGED;
- }
-
- // The current assumed state used to determine a change.
- auto AssumedState = getAssumed();
- bool Changed = false;
-
- auto CheckRWInst = [&](Instruction &I) {
- MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
- LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
- << ": " << getMemoryLocationsAsStr(MLK) << "\n");
- removeAssumedBits(inverseLocation(MLK, false, false));
- // Stop once only the valid bit set in the *not assumed location*, thus
- // once we don't actually exclude any memory locations in the state.
- return getAssumedNotAccessedLocation() != VALID_STATE;
- };
-
- bool UsedAssumedInformation = false;
- if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- Changed |= AssumedState != getAssumed();
- return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- if (isAssumedReadNone())
- STATS_DECLTRACK_FN_ATTR(readnone)
- else if (isAssumedArgMemOnly())
- STATS_DECLTRACK_FN_ATTR(argmemonly)
- else if (isAssumedInaccessibleMemOnly())
- STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
- else if (isAssumedInaccessibleOrArgMemOnly())
- STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
- }
-};
-
-/// AAMemoryLocation attribute for call sites.
-struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
- AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
- : AAMemoryLocationImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- // TODO: Once we have call site specific value information we can provide
- // call site specific liveness liveness information and then it makes
- // sense to specialize attributes for call sites arguments instead of
- // redirecting requests to the callee argument.
- Function *F = getAssociatedFunction();
- const IRPosition &FnPos = IRPosition::function(*F);
- auto *FnAA =
- A.getAAFor<AAMemoryLocation>(*this, FnPos, DepClassTy::REQUIRED);
- if (!FnAA)
- return indicatePessimisticFixpoint();
- bool Changed = false;
- auto AccessPred = [&](const Instruction *I, const Value *Ptr,
- AccessKind Kind, MemoryLocationsKind MLK) {
- updateStateAndAccessesMap(getState(), MLK, I, Ptr, Changed,
- getAccessKindFromInst(I));
- return true;
- };
- if (!FnAA->checkForAllAccessesToMemoryKind(AccessPred, ALL_LOCATIONS))
- return indicatePessimisticFixpoint();
- return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- if (isAssumedReadNone())
- STATS_DECLTRACK_CS_ATTR(readnone)
- }
-};
-} // namespace
-
-/// ------------------ denormal-fp-math Attribute -------------------------
-
-namespace {
-struct AADenormalFPMathImpl : public AADenormalFPMath {
- AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
- : AADenormalFPMath(IRP, A) {}
-
- const std::string getAsStr(Attributor *A) const override {
- std::string Str("AADenormalFPMath[");
- raw_string_ostream OS(Str);
-
- DenormalState Known = getKnown();
- if (Known.Mode.isValid())
- OS << "denormal-fp-math=" << Known.Mode;
- else
- OS << "invalid";
-
- if (Known.ModeF32.isValid())
- OS << " denormal-fp-math-f32=" << Known.ModeF32;
- OS << ']';
- return OS.str();
- }
-};
-
-struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
- AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
- : AADenormalFPMathImpl(IRP, A) {}
-
- void initialize(Attributor &A) override {
- const Function *F = getAnchorScope();
- DenormalMode Mode = F->getDenormalModeRaw();
- DenormalMode ModeF32 = F->getDenormalModeF32Raw();
-
- // TODO: Handling this here prevents handling the case where a callee has a
- // fixed denormal-fp-math with dynamic denormal-fp-math-f32, but called from
- // a function with a fully fixed mode.
- if (ModeF32 == DenormalMode::getInvalid())
- ModeF32 = Mode;
- Known = DenormalState{Mode, ModeF32};
- if (isModeFixed())
- indicateFixpoint();
- }
-
- ChangeStatus updateImpl(Attributor &A) override {
- ChangeStatus Change = ChangeStatus::UNCHANGED;
-
- auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
- Function *Caller = CS.getInstruction()->getFunction();
- LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
- << "->" << getAssociatedFunction()->getName() << '\n');
-
- const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
- *this, IRPosition::function(*Caller), DepClassTy::REQUIRED);
- if (!CallerInfo)
- return false;
-
- Change = Change | clampStateAndIndicateChange(this->getState(),
- CallerInfo->getState());
- return true;
- };
-
- bool AllCallSitesKnown = true;
- if (!A.checkForAllCallSites(CheckCallSite, *this, true, AllCallSitesKnown))
- return indicatePessimisticFixpoint();
-
- if (Change == ChangeStatus::CHANGED && isModeFixed())
- indicateFixpoint();
- return Change;
- }
-
- ChangeStatus manifest(Attributor &A) override {
- LLVMContext &Ctx = getAssociatedFunction()->getContext();
-
- SmallVector<Attribute, 2> AttrToAdd;
- SmallVector<StringRef, 2> AttrToRemove;
- if (Known.Mode == DenormalMode::getDefault()) {
- AttrToRemove.push_back("denormal-fp-math");
- } else {
- AttrToAdd.push_back(
- Attribute::get(Ctx, "denormal-fp-math", Known.Mode.str()));
- }
-
- if (Known.ModeF32 != Known.Mode) {
- AttrToAdd.push_back(
- Attribute::get(Ctx, "denormal-fp-math-f32", Known.ModeF32.str()));
- } else {
- AttrToRemove.push_back("denormal-fp-math-f32");
- }
-
- auto &IRP = getIRPosition();
-
- // TODO: There should be a combined add and remove API.
- return A.removeAttrs(IRP, AttrToRemove) |
- A.manifestAttrs(IRP, AttrToAdd, /*ForceReplace=*/true);
- }
-
- void trackStatistics() const override {
- STATS_DECLTRACK_FN_ATTR(denormal_fp_math)
- }
-};
-} // namespace
-
-/// ------------------ Value Constant Range Attribute -------------------------
-
-namespace {
-struct AAValueConstantRangeImpl : AAValueConstantRange {
- using StateType = IntegerRangeState;
- AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
- : AAValueConstantRange(IRP, A) {}
-
- /// See AbstractAttribute::initialize(..).
- void initialize(Attributor &A) override {
- if (A.hasSimplificationCallback(getIRPosition())) {
- indicatePessimisticFixpoint();
- return;
- }
-
- // Intersect a range given by SCEV.
- intersectKnown(getConstantRangeFromSCEV(A, getCtxI()));
-
- // Intersect a range given by LVI.
- intersectKnown(getConstantRangeFromLVI(A, getCtxI()));
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- std::string Str;
- llvm::raw_string_ostream OS(Str);
- OS << "range(" << getBitWidth() << ")<";
- getKnown().print(OS);
- OS << " / ";
- getAssumed().print(OS);
- OS << ">";
- return OS.str();
- }
-
- /// Helper function to get a SCEV expr for the associated value at program
- /// point \p I.
- const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
- if (!getAnchorScope())
- return nullptr;
-
- ScalarEvolution *SE =
- A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
- *getAnchorScope());
-
- LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
- *getAnchorScope());
-
- if (!SE || !LI)
- return nullptr;
-
- const SCEV *S = SE->getSCEV(&getAssociatedValue());
- if (!I)
- return S;
-
- return SE->getSCEVAtScope(S, LI->getLoopFor(I->getParent()));
- }
-
- /// Helper function to get a range from SCEV for the associated value at
- /// program point \p I.
- ConstantRange getConstantRangeFromSCEV(Attributor &A,
- const Instruction *I = nullptr) const {
- if (!getAnchorScope())
- return getWorstState(getBitWidth());
-
- ScalarEvolution *SE =
- A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
- *getAnchorScope());
-
- const SCEV *S = getSCEV(A, I);
- if (!SE || !S)
- return getWorstState(getBitWidth());
-
- return SE->getUnsignedRange(S);
- }
-
- /// Helper function to get a range from LVI for the associated value at
- /// program point \p I.
- ConstantRange
- getConstantRangeFromLVI(Attributor &A,
- const Instruction *CtxI = nullptr) const {
- if (!getAnchorScope())
- return getWorstState(getBitWidth());
-
- LazyValueInfo *LVI =
- A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
- *getAnchorScope());
-
- if (!LVI || !CtxI)
- return getWorstState(getBitWidth());
- return LVI->getConstantRange(&getAssociatedValue(),
- const_cast<Instruction *>(CtxI),
- /*UndefAllowed*/ false);
- }
-
- /// Return true if \p CtxI is valid for querying outside analyses.
- /// This basically makes sure we do not ask intra-procedural analysis
- /// about a context in the wrong function or a context that violates
- /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
- /// if the original context of this AA is OK or should be considered invalid.
- bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
- const Instruction *CtxI,
- bool AllowAACtxI) const {
- if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
- return false;
-
- // Our context might be in a different function, neither intra-procedural
- // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
- if (!AA::isValidInScope(getAssociatedValue(), CtxI->getFunction()))
- return false;
-
- // If the context is not dominated by the value there are paths to the
- // context that do not define the value. This cannot be handled by
- // LazyValueInfo so we need to bail.
- if (auto *I = dyn_cast<Instruction>(&getAssociatedValue())) {
- InformationCache &InfoCache = A.getInfoCache();
- const DominatorTree *DT =
- InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
- *I->getFunction());
- return DT && DT->dominates(I, CtxI);
- }
-
- return true;
- }
-
- /// See AAValueConstantRange::getKnownConstantRange(..).
- ConstantRange
- getKnownConstantRange(Attributor &A,
- const Instruction *CtxI = nullptr) const override {
- if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
- /* AllowAACtxI */ false))
- return getKnown();
-
- ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
- ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);
- return getKnown().intersectWith(SCEVR).intersectWith(LVIR);
- }
-
- /// See AAValueConstantRange::getAssumedConstantRange(..).
- ConstantRange
- getAssumedConstantRange(Attributor &A,
- const Instruction *CtxI = nullptr) const override {
- // TODO: Make SCEV use Attributor assumption.
- // We may be able to bound a variable range via assumptions in
- // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
- // evolve to x^2 + x, then we can say that y is in [2, 12].
- if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
- /* AllowAACtxI */ false))
- return getAssumed();
-
- ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
- ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);
- return getAssumed().intersectWith(SCEVR).intersectWith(LVIR);
- }
-
- /// Helper function to create MDNode for range metadata.
- static MDNode *
- getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
- const ConstantRange &AssumedConstantRange) {
- Metadata *LowAndHigh[] = {ConstantAsMetadata::get(ConstantInt::get(
- Ty, AssumedConstantRange.getLower())),
- ConstantAsMetadata::get(ConstantInt::get(
- Ty, AssumedConstantRange.getUpper()))};
- return MDNode::get(Ctx, LowAndHigh);
- }
-
- /// Return true if \p Assumed is included in \p KnownRanges.
- static bool isBetterRange(const ConstantRange &Assumed, MDNode *KnownRanges) {
-
- if (Assumed.isFullSet())
- return false;
-
- if (!KnownRanges)
- return true;
-
- // If multiple ranges are annotated in IR, we give up to annotate assumed
- // range for now.
-
- // TODO: If there exists a known range which containts assumed range, we
- // can say assumed range is better.
- if (KnownRanges->getNumOperands() > 2)
- return false;
-
- ConstantInt *Lower =
- mdconst::extract<ConstantInt>(KnownRanges->getOperand(0));
- ConstantInt *Upper =
- mdconst::extract<ConstantInt>(KnownRanges->getOperand(1));
-
- ConstantRange Known(Lower->getValue(), Upper->getValue());
- return Known.contains(Assumed) && Known != Assumed;
- }
-
- /// Helper function to set range metadata.
- static bool
- setRangeMetadataIfisBetterRange(Instruction *I,
- const ConstantRange &AssumedConstantRange) {
- auto *OldRangeMD = I->getMetadata(LLVMContext::MD_range);
- if (isBetterRange(AssumedConstantRange, OldRangeMD)) {
- if (!AssumedConstantRange.isEmptySet()) {
- I->setMetadata(LLVMContext::MD_range,
- getMDNodeForConstantRange(I->getType(), I->getContext(),
- AssumedConstantRange));
- return true;
- }
- }
- return false;
- }
-
- /// See AbstractAttribute::manifest()
- ChangeStatus manifest(Attributor &A) override {
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
- assert(!AssumedConstantRange.isFullSet() && "Invalid state");
-
- auto &V = getAssociatedValue();
- if (!AssumedConstantRange.isEmptySet() &&
- !AssumedConstantRange.isSingleElement()) {
- if (Instruction *I = dyn_cast<Instruction>(&V)) {
- assert(I == getCtxI() && "Should not annotate an instruction which is "
- "not the context instruction");
- if (isa<CallInst>(I) || isa<LoadInst>(I))
- if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
- Changed = ChangeStatus::CHANGED;
- }
- }
-
- return Changed;
- }
-};
-
-struct AAValueConstantRangeArgument final
- : AAArgumentFromCallSiteArguments<
- AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
- true /* BridgeCallBaseContext */> {
- using Base = AAArgumentFromCallSiteArguments<
- AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
- true /* BridgeCallBaseContext */>;
- AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_ARG_ATTR(value_range)
- }
-};
-
-struct AAValueConstantRangeReturned
- : AAReturnedFromReturnedValues<AAValueConstantRange,
- AAValueConstantRangeImpl,
- AAValueConstantRangeImpl::StateType,
- /* PropogateCallBaseContext */ true> {
- using Base =
- AAReturnedFromReturnedValues<AAValueConstantRange,
- AAValueConstantRangeImpl,
- AAValueConstantRangeImpl::StateType,
- /* PropogateCallBaseContext */ true>;
- AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))
- indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FNRET_ATTR(value_range)
- }
-};
-
-struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
- AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
- : AAValueConstantRangeImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- AAValueConstantRangeImpl::initialize(A);
- if (isAtFixpoint())
- return;
-
- Value &V = getAssociatedValue();
-
- if (auto *C = dyn_cast<ConstantInt>(&V)) {
- unionAssumed(ConstantRange(C->getValue()));
- indicateOptimisticFixpoint();
- return;
- }
-
- if (isa<UndefValue>(&V)) {
- // Collapse the undef state to 0.
- unionAssumed(ConstantRange(APInt(getBitWidth(), 0)));
- indicateOptimisticFixpoint();
- return;
- }
-
- if (isa<CallBase>(&V))
- return;
-
- if (isa<BinaryOperator>(&V) || isa<CmpInst>(&V) || isa<CastInst>(&V))
- return;
-
- // If it is a load instruction with range metadata, use it.
- if (LoadInst *LI = dyn_cast<LoadInst>(&V))
- if (auto *RangeMD = LI->getMetadata(LLVMContext::MD_range)) {
- intersectKnown(getConstantRangeFromMetadata(*RangeMD));
- return;
- }
-
- // We can work with PHI and select instruction as we traverse their operands
- // during update.
- if (isa<SelectInst>(V) || isa<PHINode>(V))
- return;
-
- // Otherwise we give up.
- indicatePessimisticFixpoint();
-
- LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
- << getAssociatedValue() << "\n");
- }
-
- bool calculateBinaryOperator(
- Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
- const Instruction *CtxI,
- SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
- Value *LHS = BinOp->getOperand(0);
- Value *RHS = BinOp->getOperand(1);
-
- // Simplify the operands first.
- bool UsedAssumedInformation = false;
- const auto &SimplifiedLHS = A.getAssumedSimplified(
- IRPosition::value(*LHS, getCallBaseContext()), *this,
- UsedAssumedInformation, AA::Interprocedural);
- if (!SimplifiedLHS.has_value())
- return true;
- if (!*SimplifiedLHS)
- return false;
- LHS = *SimplifiedLHS;
-
- const auto &SimplifiedRHS = A.getAssumedSimplified(
- IRPosition::value(*RHS, getCallBaseContext()), *this,
- UsedAssumedInformation, AA::Interprocedural);
- if (!SimplifiedRHS.has_value())
- return true;
- if (!*SimplifiedRHS)
- return false;
- RHS = *SimplifiedRHS;
-
- // TODO: Allow non integers as well.
- if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
- return false;
-
- auto *LHSAA = A.getAAFor<AAValueConstantRange>(
- *this, IRPosition::value(*LHS, getCallBaseContext()),
- DepClassTy::REQUIRED);
- if (!LHSAA)
- return false;
- QuerriedAAs.push_back(LHSAA);
- auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
-
- auto *RHSAA = A.getAAFor<AAValueConstantRange>(
- *this, IRPosition::value(*RHS, getCallBaseContext()),
- DepClassTy::REQUIRED);
- if (!RHSAA)
- return false;
- QuerriedAAs.push_back(RHSAA);
- auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
-
- auto AssumedRange = LHSAARange.binaryOp(BinOp->getOpcode(), RHSAARange);
-
- T.unionAssumed(AssumedRange);
-
- // TODO: Track a known state too.
-
- return T.isValidState();
- }
-
- bool calculateCastInst(
- Attributor &A, CastInst *CastI, IntegerRangeState &T,
- const Instruction *CtxI,
- SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
- assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
- // TODO: Allow non integers as well.
- Value *OpV = CastI->getOperand(0);
-
- // Simplify the operand first.
- bool UsedAssumedInformation = false;
- const auto &SimplifiedOpV = A.getAssumedSimplified(
- IRPosition::value(*OpV, getCallBaseContext()), *this,
- UsedAssumedInformation, AA::Interprocedural);
- if (!SimplifiedOpV.has_value())
- return true;
- if (!*SimplifiedOpV)
- return false;
- OpV = *SimplifiedOpV;
-
- if (!OpV->getType()->isIntegerTy())
- return false;
-
- auto *OpAA = A.getAAFor<AAValueConstantRange>(
- *this, IRPosition::value(*OpV, getCallBaseContext()),
- DepClassTy::REQUIRED);
- if (!OpAA)
- return false;
- QuerriedAAs.push_back(OpAA);
- T.unionAssumed(OpAA->getAssumed().castOp(CastI->getOpcode(),
- getState().getBitWidth()));
- return T.isValidState();
- }
-
- bool
- calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
- const Instruction *CtxI,
- SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
- Value *LHS = CmpI->getOperand(0);
- Value *RHS = CmpI->getOperand(1);
-
- // Simplify the operands first.
- bool UsedAssumedInformation = false;
- const auto &SimplifiedLHS = A.getAssumedSimplified(
- IRPosition::value(*LHS, getCallBaseContext()), *this,
- UsedAssumedInformation, AA::Interprocedural);
- if (!SimplifiedLHS.has_value())
- return true;
- if (!*SimplifiedLHS)
- return false;
- LHS = *SimplifiedLHS;
-
- const auto &SimplifiedRHS = A.getAssumedSimplified(
- IRPosition::value(*RHS, getCallBaseContext()), *this,
- UsedAssumedInformation, AA::Interprocedural);
- if (!SimplifiedRHS.has_value())
- return true;
- if (!*SimplifiedRHS)
- return false;
- RHS = *SimplifiedRHS;
-
- // TODO: Allow non integers as well.
- if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
- return false;
-
- auto *LHSAA = A.getAAFor<AAValueConstantRange>(
- *this, IRPosition::value(*LHS, getCallBaseContext()),
- DepClassTy::REQUIRED);
- if (!LHSAA)
- return false;
- QuerriedAAs.push_back(LHSAA);
- auto *RHSAA = A.getAAFor<AAValueConstantRange>(
- *this, IRPosition::value(*RHS, getCallBaseContext()),
- DepClassTy::REQUIRED);
- if (!RHSAA)
- return false;
- QuerriedAAs.push_back(RHSAA);
- auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
- auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
-
- // If one of them is empty set, we can't decide.
- if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
- return true;
-
- bool MustTrue = false, MustFalse = false;
-
- auto AllowedRegion =
- ConstantRange::makeAllowedICmpRegion(CmpI->getPredicate(), RHSAARange);
-
- if (AllowedRegion.intersectWith(LHSAARange).isEmptySet())
- MustFalse = true;
-
- if (LHSAARange.icmp(CmpI->getPredicate(), RHSAARange))
- MustTrue = true;
-
- assert((!MustTrue || !MustFalse) &&
- "Either MustTrue or MustFalse should be false!");
-
- if (MustTrue)
- T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
- else if (MustFalse)
- T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
- else
- T.unionAssumed(ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
-
- LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
- << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
- << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
- << *RHSAA);
-
- // TODO: Track a known state too.
- return T.isValidState();
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
-
- IntegerRangeState T(getBitWidth());
- auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
- Instruction *I = dyn_cast<Instruction>(&V);
- if (!I || isa<CallBase>(I)) {
-
- // Simplify the operand first.
- bool UsedAssumedInformation = false;
- const auto &SimplifiedOpV = A.getAssumedSimplified(
- IRPosition::value(V, getCallBaseContext()), *this,
- UsedAssumedInformation, AA::Interprocedural);
- if (!SimplifiedOpV.has_value())
- return true;
- if (!*SimplifiedOpV)
- return false;
- Value *VPtr = *SimplifiedOpV;
-
- // If the value is not instruction, we query AA to Attributor.
- const auto *AA = A.getAAFor<AAValueConstantRange>(
- *this, IRPosition::value(*VPtr, getCallBaseContext()),
- DepClassTy::REQUIRED);
-
- // Clamp operator is not used to utilize a program point CtxI.
- if (AA)
- T.unionAssumed(AA->getAssumedConstantRange(A, CtxI));
- else
- return false;
-
- return T.isValidState();
- }
-
- SmallVector<const AAValueConstantRange *, 4> QuerriedAAs;
- if (auto *BinOp = dyn_cast<BinaryOperator>(I)) {
- if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
- return false;
- } else if (auto *CmpI = dyn_cast<CmpInst>(I)) {
- if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
- return false;
- } else if (auto *CastI = dyn_cast<CastInst>(I)) {
- if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
- return false;
- } else {
- // Give up with other instructions.
- // TODO: Add other instructions
-
- T.indicatePessimisticFixpoint();
- return false;
- }
-
- // Catch circular reasoning in a pessimistic way for now.
- // TODO: Check how the range evolves and if we stripped anything, see also
- // AADereferenceable or AAAlign for similar situations.
- for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
- if (QueriedAA != this)
- continue;
- // If we are in a stady state we do not need to worry.
- if (T.getAssumed() == getState().getAssumed())
- continue;
- T.indicatePessimisticFixpoint();
- }
-
- return T.isValidState();
- };
-
- if (!VisitValueCB(getAssociatedValue(), getCtxI()))
- return indicatePessimisticFixpoint();
-
- // Ensure that long def-use chains can't cause circular reasoning either by
- // introducing a cutoff below.
- if (clampStateAndIndicateChange(getState(), T) == ChangeStatus::UNCHANGED)
- return ChangeStatus::UNCHANGED;
- if (++NumChanges > MaxNumChanges) {
- LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
- << " but only " << MaxNumChanges
- << " are allowed to avoid cyclic reasoning.");
- return indicatePessimisticFixpoint();
- }
- return ChangeStatus::CHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(value_range)
- }
-
- /// Tracker to bail after too many widening steps of the constant range.
- int NumChanges = 0;
-
- /// Upper bound for the number of allowed changes (=widening steps) for the
- /// constant range before we give up.
- static constexpr int MaxNumChanges = 5;
-};
-
-struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
- AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
- : AAValueConstantRangeImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- ChangeStatus updateImpl(Attributor &A) override {
- llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
- "not be called");
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
-};
-
-struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
- AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
- : AAValueConstantRangeFunction(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
-};
-
-struct AAValueConstantRangeCallSiteReturned
- : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
- AAValueConstantRangeImpl::StateType,
- /* IntroduceCallBaseContext */ true> {
- AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
- AAValueConstantRangeImpl::StateType,
- /* IntroduceCallBaseContext */ true>(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- // If it is a load instruction with range metadata, use the metadata.
- if (CallInst *CI = dyn_cast<CallInst>(&getAssociatedValue()))
- if (auto *RangeMD = CI->getMetadata(LLVMContext::MD_range))
- intersectKnown(getConstantRangeFromMetadata(*RangeMD));
-
- AAValueConstantRangeImpl::initialize(A);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSRET_ATTR(value_range)
- }
-};
-struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
- AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAValueConstantRangeFloating(IRP, A) {}
-
- /// See AbstractAttribute::manifest()
- ChangeStatus manifest(Attributor &A) override {
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSARG_ATTR(value_range)
- }
-};
-} // namespace
-
-/// ------------------ Potential Values Attribute -------------------------
-
-namespace {
-struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
- using StateType = PotentialConstantIntValuesState;
-
- AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
- : AAPotentialConstantValues(IRP, A) {}
-
- /// See AbstractAttribute::initialize(..).
- void initialize(Attributor &A) override {
- if (A.hasSimplificationCallback(getIRPosition()))
- indicatePessimisticFixpoint();
- else
- AAPotentialConstantValues::initialize(A);
- }
-
- bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
- bool &ContainsUndef, bool ForSelf) {
- SmallVector<AA::ValueAndContext> Values;
- bool UsedAssumedInformation = false;
- if (!A.getAssumedSimplifiedValues(IRP, *this, Values, AA::Interprocedural,
- UsedAssumedInformation)) {
- // Avoid recursion when the caller is computing constant values for this
- // IRP itself.
- if (ForSelf)
- return false;
- if (!IRP.getAssociatedType()->isIntegerTy())
- return false;
- auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
- *this, IRP, DepClassTy::REQUIRED);
- if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
- return false;
- ContainsUndef = PotentialValuesAA->getState().undefIsContained();
- S = PotentialValuesAA->getState().getAssumedSet();
- return true;
- }
-
- // Copy all the constant values, except UndefValue. ContainsUndef is true
- // iff Values contains only UndefValue instances. If there are other known
- // constants, then UndefValue is dropped.
- ContainsUndef = false;
- for (auto &It : Values) {
- if (isa<UndefValue>(It.getValue())) {
- ContainsUndef = true;
- continue;
- }
- auto *CI = dyn_cast<ConstantInt>(It.getValue());
- if (!CI)
- return false;
- S.insert(CI->getValue());
- }
- ContainsUndef &= S.empty();
-
- return true;
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- std::string Str;
- llvm::raw_string_ostream OS(Str);
- OS << getState();
- return OS.str();
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- return indicatePessimisticFixpoint();
- }
-};
-
-struct AAPotentialConstantValuesArgument final
- : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
- AAPotentialConstantValuesImpl,
- PotentialConstantIntValuesState> {
- using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
- AAPotentialConstantValuesImpl,
- PotentialConstantIntValuesState>;
- AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_ARG_ATTR(potential_values)
- }
-};
-
-struct AAPotentialConstantValuesReturned
- : AAReturnedFromReturnedValues<AAPotentialConstantValues,
- AAPotentialConstantValuesImpl> {
- using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
- AAPotentialConstantValuesImpl>;
- AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- void initialize(Attributor &A) override {
- if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))
- indicatePessimisticFixpoint();
- Base::initialize(A);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FNRET_ATTR(potential_values)
- }
-};
-
-struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
- AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
- : AAPotentialConstantValuesImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(..).
- void initialize(Attributor &A) override {
- AAPotentialConstantValuesImpl::initialize(A);
- if (isAtFixpoint())
- return;
-
- Value &V = getAssociatedValue();
-
- if (auto *C = dyn_cast<ConstantInt>(&V)) {
- unionAssumed(C->getValue());
- indicateOptimisticFixpoint();
- return;
- }
-
- if (isa<UndefValue>(&V)) {
- unionAssumedWithUndef();
- indicateOptimisticFixpoint();
- return;
- }
-
- if (isa<BinaryOperator>(&V) || isa<ICmpInst>(&V) || isa<CastInst>(&V))
- return;
-
- if (isa<SelectInst>(V) || isa<PHINode>(V) || isa<LoadInst>(V))
- return;
-
- indicatePessimisticFixpoint();
-
- LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
- << getAssociatedValue() << "\n");
- }
-
- static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
- const APInt &RHS) {
- return ICmpInst::compare(LHS, RHS, ICI->getPredicate());
- }
-
- static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
- uint32_t ResultBitWidth) {
- Instruction::CastOps CastOp = CI->getOpcode();
- switch (CastOp) {
- default:
- llvm_unreachable("unsupported or not integer cast");
- case Instruction::Trunc:
- return Src.trunc(ResultBitWidth);
- case Instruction::SExt:
- return Src.sext(ResultBitWidth);
- case Instruction::ZExt:
- return Src.zext(ResultBitWidth);
- case Instruction::BitCast:
- return Src;
- }
- }
-
- static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
- const APInt &LHS, const APInt &RHS,
- bool &SkipOperation, bool &Unsupported) {
- Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
- // Unsupported is set to true when the binary operator is not supported.
- // SkipOperation is set to true when UB occur with the given operand pair
- // (LHS, RHS).
- // TODO: we should look at nsw and nuw keywords to handle operations
- // that create poison or undef value.
- switch (BinOpcode) {
- default:
- Unsupported = true;
- return LHS;
- case Instruction::Add:
- return LHS + RHS;
- case Instruction::Sub:
- return LHS - RHS;
- case Instruction::Mul:
- return LHS * RHS;
- case Instruction::UDiv:
- if (RHS.isZero()) {
- SkipOperation = true;
- return LHS;
- }
- return LHS.udiv(RHS);
- case Instruction::SDiv:
- if (RHS.isZero()) {
- SkipOperation = true;
- return LHS;
- }
- return LHS.sdiv(RHS);
- case Instruction::URem:
- if (RHS.isZero()) {
- SkipOperation = true;
- return LHS;
- }
- return LHS.urem(RHS);
- case Instruction::SRem:
- if (RHS.isZero()) {
- SkipOperation = true;
- return LHS;
- }
- return LHS.srem(RHS);
- case Instruction::Shl:
- return LHS.shl(RHS);
- case Instruction::LShr:
- return LHS.lshr(RHS);
- case Instruction::AShr:
- return LHS.ashr(RHS);
- case Instruction::And:
- return LHS & RHS;
- case Instruction::Or:
- return LHS | RHS;
- case Instruction::Xor:
- return LHS ^ RHS;
- }
- }
-
- bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
- const APInt &LHS, const APInt &RHS) {
- bool SkipOperation = false;
- bool Unsupported = false;
- APInt Result =
- calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
- if (Unsupported)
- return false;
- // If SkipOperation is true, we can ignore this operand pair (L, R).
- if (!SkipOperation)
- unionAssumed(Result);
- return isValidState();
- }
-
- ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
- auto AssumedBefore = getAssumed();
- Value *LHS = ICI->getOperand(0);
- Value *RHS = ICI->getOperand(1);
-
- bool LHSContainsUndef = false, RHSContainsUndef = false;
- SetTy LHSAAPVS, RHSAAPVS;
- if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
- LHSContainsUndef, /* ForSelf */ false) ||
- !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
- RHSContainsUndef, /* ForSelf */ false))
- return indicatePessimisticFixpoint();
-
- // TODO: make use of undef flag to limit potential values aggressively.
- bool MaybeTrue = false, MaybeFalse = false;
- const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
- if (LHSContainsUndef && RHSContainsUndef) {
- // The result of any comparison between undefs can be soundly replaced
- // with undef.
- unionAssumedWithUndef();
- } else if (LHSContainsUndef) {
- for (const APInt &R : RHSAAPVS) {
- bool CmpResult = calculateICmpInst(ICI, Zero, R);
- MaybeTrue |= CmpResult;
- MaybeFalse |= !CmpResult;
- if (MaybeTrue & MaybeFalse)
- return indicatePessimisticFixpoint();
- }
- } else if (RHSContainsUndef) {
- for (const APInt &L : LHSAAPVS) {
- bool CmpResult = calculateICmpInst(ICI, L, Zero);
- MaybeTrue |= CmpResult;
- MaybeFalse |= !CmpResult;
- if (MaybeTrue & MaybeFalse)
- return indicatePessimisticFixpoint();
- }
- } else {
- for (const APInt &L : LHSAAPVS) {
- for (const APInt &R : RHSAAPVS) {
- bool CmpResult = calculateICmpInst(ICI, L, R);
- MaybeTrue |= CmpResult;
- MaybeFalse |= !CmpResult;
- if (MaybeTrue & MaybeFalse)
- return indicatePessimisticFixpoint();
- }
- }
- }
- if (MaybeTrue)
- unionAssumed(APInt(/* numBits */ 1, /* val */ 1));
- if (MaybeFalse)
- unionAssumed(APInt(/* numBits */ 1, /* val */ 0));
- return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
- auto AssumedBefore = getAssumed();
- Value *LHS = SI->getTrueValue();
- Value *RHS = SI->getFalseValue();
-
- bool UsedAssumedInformation = false;
- std::optional<Constant *> C = A.getAssumedConstant(
- *SI->getCondition(), *this, UsedAssumedInformation);
-
- // Check if we only need one operand.
- bool OnlyLeft = false, OnlyRight = false;
- if (C && *C && (*C)->isOneValue())
- OnlyLeft = true;
- else if (C && *C && (*C)->isZeroValue())
- OnlyRight = true;
-
- bool LHSContainsUndef = false, RHSContainsUndef = false;
- SetTy LHSAAPVS, RHSAAPVS;
- if (!OnlyRight &&
- !fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
- LHSContainsUndef, /* ForSelf */ false))
- return indicatePessimisticFixpoint();
-
- if (!OnlyLeft &&
- !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
- RHSContainsUndef, /* ForSelf */ false))
- return indicatePessimisticFixpoint();
-
- if (OnlyLeft || OnlyRight) {
- // select (true/false), lhs, rhs
- auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
- auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
-
- if (Undef)
- unionAssumedWithUndef();
- else {
- for (const auto &It : *OpAA)
- unionAssumed(It);
- }
-
- } else if (LHSContainsUndef && RHSContainsUndef) {
- // select i1 *, undef , undef => undef
- unionAssumedWithUndef();
- } else {
- for (const auto &It : LHSAAPVS)
- unionAssumed(It);
- for (const auto &It : RHSAAPVS)
- unionAssumed(It);
- }
- return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
- auto AssumedBefore = getAssumed();
- if (!CI->isIntegerCast())
- return indicatePessimisticFixpoint();
- assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
- uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
- Value *Src = CI->getOperand(0);
-
- bool SrcContainsUndef = false;
- SetTy SrcPVS;
- if (!fillSetWithConstantValues(A, IRPosition::value(*Src), SrcPVS,
- SrcContainsUndef, /* ForSelf */ false))
- return indicatePessimisticFixpoint();
-
- if (SrcContainsUndef)
- unionAssumedWithUndef();
- else {
- for (const APInt &S : SrcPVS) {
- APInt T = calculateCastInst(CI, S, ResultBitWidth);
- unionAssumed(T);
- }
- }
- return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
- auto AssumedBefore = getAssumed();
- Value *LHS = BinOp->getOperand(0);
- Value *RHS = BinOp->getOperand(1);
-
- bool LHSContainsUndef = false, RHSContainsUndef = false;
- SetTy LHSAAPVS, RHSAAPVS;
- if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
- LHSContainsUndef, /* ForSelf */ false) ||
- !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
- RHSContainsUndef, /* ForSelf */ false))
- return indicatePessimisticFixpoint();
-
- const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
-
- // TODO: make use of undef flag to limit potential values aggressively.
- if (LHSContainsUndef && RHSContainsUndef) {
- if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, Zero))
- return indicatePessimisticFixpoint();
- } else if (LHSContainsUndef) {
- for (const APInt &R : RHSAAPVS) {
- if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, R))
- return indicatePessimisticFixpoint();
- }
- } else if (RHSContainsUndef) {
- for (const APInt &L : LHSAAPVS) {
- if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, Zero))
- return indicatePessimisticFixpoint();
- }
- } else {
- for (const APInt &L : LHSAAPVS) {
- for (const APInt &R : RHSAAPVS) {
- if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, R))
- return indicatePessimisticFixpoint();
- }
- }
- }
- return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
- auto AssumedBefore = getAssumed();
- SetTy Incoming;
- bool ContainsUndef;
- if (!fillSetWithConstantValues(A, IRPosition::value(*Inst), Incoming,
- ContainsUndef, /* ForSelf */ true))
- return indicatePessimisticFixpoint();
- if (ContainsUndef) {
- unionAssumedWithUndef();
- } else {
- for (const auto &It : Incoming)
- unionAssumed(It);
- }
- return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- Value &V = getAssociatedValue();
- Instruction *I = dyn_cast<Instruction>(&V);
-
- if (auto *ICI = dyn_cast<ICmpInst>(I))
- return updateWithICmpInst(A, ICI);
-
- if (auto *SI = dyn_cast<SelectInst>(I))
- return updateWithSelectInst(A, SI);
-
- if (auto *CI = dyn_cast<CastInst>(I))
- return updateWithCastInst(A, CI);
-
- if (auto *BinOp = dyn_cast<BinaryOperator>(I))
- return updateWithBinaryOperator(A, BinOp);
-
- if (isa<PHINode>(I) || isa<LoadInst>(I))
- return updateWithInstruction(A, I);
-
- return indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(potential_values)
- }
-};
-
-struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
- AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
- : AAPotentialConstantValuesImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- ChangeStatus updateImpl(Attributor &A) override {
- llvm_unreachable(
- "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
- "not be called");
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FN_ATTR(potential_values)
- }
-};
-
-struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
- AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
- : AAPotentialConstantValuesFunction(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CS_ATTR(potential_values)
- }
-};
-
-struct AAPotentialConstantValuesCallSiteReturned
- : AACalleeToCallSite<AAPotentialConstantValues,
- AAPotentialConstantValuesImpl> {
- AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
- Attributor &A)
- : AACalleeToCallSite<AAPotentialConstantValues,
- AAPotentialConstantValuesImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSRET_ATTR(potential_values)
- }
-};
-
-struct AAPotentialConstantValuesCallSiteArgument
- : AAPotentialConstantValuesFloating {
- AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
- Attributor &A)
- : AAPotentialConstantValuesFloating(IRP, A) {}
-
- /// See AbstractAttribute::initialize(..).
- void initialize(Attributor &A) override {
- AAPotentialConstantValuesImpl::initialize(A);
- if (isAtFixpoint())
- return;
-
- Value &V = getAssociatedValue();
-
- if (auto *C = dyn_cast<ConstantInt>(&V)) {
- unionAssumed(C->getValue());
- indicateOptimisticFixpoint();
- return;
- }
-
- if (isa<UndefValue>(&V)) {
- unionAssumedWithUndef();
- indicateOptimisticFixpoint();
- return;
- }
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- Value &V = getAssociatedValue();
- auto AssumedBefore = getAssumed();
- auto *AA = A.getAAFor<AAPotentialConstantValues>(
- *this, IRPosition::value(V), DepClassTy::REQUIRED);
- if (!AA)
- return indicatePessimisticFixpoint();
- const auto &S = AA->getAssumed();
- unionAssumed(S);
- return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSARG_ATTR(potential_values)
- }
-};
-} // namespace
-
-/// ------------------------ NoUndef Attribute ---------------------------------
-bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP,
- Attribute::AttrKind ImpliedAttributeKind,
- bool IgnoreSubsumingPositions) {
- assert(ImpliedAttributeKind == Attribute::NoUndef &&
- "Unexpected attribute kind");
- if (A.hasAttr(IRP, {Attribute::NoUndef}, IgnoreSubsumingPositions,
- Attribute::NoUndef))
- return true;
-
- Value &Val = IRP.getAssociatedValue();
- if (IRP.getPositionKind() != IRPosition::IRP_RETURNED &&
- isGuaranteedNotToBeUndefOrPoison(&Val)) {
- LLVMContext &Ctx = Val.getContext();
- A.manifestAttrs(IRP, Attribute::get(Ctx, Attribute::NoUndef));
- return true;
- }
-
- return false;
-}
-
-namespace {
-struct AANoUndefImpl : AANoUndef {
- AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- Value &V = getAssociatedValue();
- if (isa<UndefValue>(V))
- indicatePessimisticFixpoint();
- assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
- }
-
- /// See followUsesInMBEC
- bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
- AANoUndef::StateType &State) {
- const Value *UseV = U->get();
- const DominatorTree *DT = nullptr;
- AssumptionCache *AC = nullptr;
- InformationCache &InfoCache = A.getInfoCache();
- if (Function *F = getAnchorScope()) {
- DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);
- AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
- }
- State.setKnown(isGuaranteedNotToBeUndefOrPoison(UseV, AC, I, DT));
- bool TrackUse = false;
- // Track use for instructions which must produce undef or poison bits when
- // at least one operand contains such bits.
- if (isa<CastInst>(*I) || isa<GetElementPtrInst>(*I))
- TrackUse = true;
- return TrackUse;
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return getAssumed() ? "noundef" : "may-undef-or-poison";
- }
-
- ChangeStatus manifest(Attributor &A) override {
- // We don't manifest noundef attribute for dead positions because the
- // associated values with dead positions would be replaced with undef
- // values.
- bool UsedAssumedInformation = false;
- if (A.isAssumedDead(getIRPosition(), nullptr, nullptr,
- UsedAssumedInformation))
- return ChangeStatus::UNCHANGED;
- // A position whose simplified value does not have any value is
- // considered to be dead. We don't manifest noundef in such positions for
- // the same reason above.
- if (!A.getAssumedSimplified(getIRPosition(), *this, UsedAssumedInformation,
- AA::Interprocedural)
- .has_value())
- return ChangeStatus::UNCHANGED;
- return AANoUndef::manifest(A);
- }
-};
-
-struct AANoUndefFloating : public AANoUndefImpl {
- AANoUndefFloating(const IRPosition &IRP, Attributor &A)
- : AANoUndefImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- AANoUndefImpl::initialize(A);
- if (!getState().isAtFixpoint() && getAnchorScope() &&
- !getAnchorScope()->isDeclaration())
- if (Instruction *CtxI = getCtxI())
- followUsesInMBEC(*this, A, getState(), *CtxI);
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
- bool IsKnownNoUndef;
- return AA::hasAssumedIRAttr<Attribute::NoUndef>(
- A, this, IRP, DepClassTy::REQUIRED, IsKnownNoUndef);
- };
-
- bool Stripped;
- bool UsedAssumedInformation = false;
- Value *AssociatedValue = &getAssociatedValue();
- SmallVector<AA::ValueAndContext> Values;
- if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
- AA::AnyScope, UsedAssumedInformation))
- Stripped = false;
- else
- Stripped =
- Values.size() != 1 || Values.front().getValue() != AssociatedValue;
-
- if (!Stripped) {
- // If we haven't stripped anything we might still be able to use a
- // different AA, but only if the IRP changes. Effectively when we
- // interpret this not as a call site value but as a floating/argument
- // value.
- const IRPosition AVIRP = IRPosition::value(*AssociatedValue);
- if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
- return indicatePessimisticFixpoint();
- return ChangeStatus::UNCHANGED;
- }
-
- for (const auto &VAC : Values)
- if (!VisitValueCB(IRPosition::value(*VAC.getValue())))
- return indicatePessimisticFixpoint();
-
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
-};
-
-struct AANoUndefReturned final
- : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
- AANoUndefReturned(const IRPosition &IRP, Attributor &A)
- : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
-};
-
-struct AANoUndefArgument final
- : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
- AANoUndefArgument(const IRPosition &IRP, Attributor &A)
- : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
-};
-
-struct AANoUndefCallSiteArgument final : AANoUndefFloating {
- AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AANoUndefFloating(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
-};
-
-struct AANoUndefCallSiteReturned final
- : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
- AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
-};
-
-/// ------------------------ NoFPClass Attribute -------------------------------
-
-struct AANoFPClassImpl : AANoFPClass {
- AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
-
- void initialize(Attributor &A) override {
- const IRPosition &IRP = getIRPosition();
-
- Value &V = IRP.getAssociatedValue();
- if (isa<UndefValue>(V)) {
- indicateOptimisticFixpoint();
- return;
- }
-
- SmallVector<Attribute> Attrs;
- A.getAttrs(getIRPosition(), {Attribute::NoFPClass}, Attrs, false);
- for (const auto &Attr : Attrs) {
- addKnownBits(Attr.getNoFPClass());
- }
-
- const DataLayout &DL = A.getDataLayout();
- if (getPositionKind() != IRPosition::IRP_RETURNED) {
- KnownFPClass KnownFPClass = computeKnownFPClass(&V, DL);
- addKnownBits(~KnownFPClass.KnownFPClasses);
- }
-
- if (Instruction *CtxI = getCtxI())
- followUsesInMBEC(*this, A, getState(), *CtxI);
- }
-
- /// See followUsesInMBEC
- bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
- AANoFPClass::StateType &State) {
- const Value *UseV = U->get();
- const DominatorTree *DT = nullptr;
- AssumptionCache *AC = nullptr;
- const TargetLibraryInfo *TLI = nullptr;
- InformationCache &InfoCache = A.getInfoCache();
-
- if (Function *F = getAnchorScope()) {
- DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);
- AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
- TLI = InfoCache.getTargetLibraryInfoForFunction(*F);
- }
-
- const DataLayout &DL = A.getDataLayout();
-
- KnownFPClass KnownFPClass =
- computeKnownFPClass(UseV, DL,
- /*InterestedClasses=*/fcAllFlags,
- /*Depth=*/0, TLI, AC, I, DT);
- State.addKnownBits(~KnownFPClass.KnownFPClasses);
-
- if (auto *CI = dyn_cast<CallInst>(UseV)) {
- // Special case FP intrinsic with struct return type.
- switch (CI->getIntrinsicID()) {
- case Intrinsic::frexp:
- return true;
- case Intrinsic::not_intrinsic:
- // TODO: Could recognize math libcalls
- return false;
- default:
- break;
- }
- }
-
- if (!UseV->getType()->isFPOrFPVectorTy())
- return false;
- return !isa<LoadInst, AtomicRMWInst>(UseV);
- }
-
- const std::string getAsStr(Attributor *A) const override {
- std::string Result = "nofpclass";
- raw_string_ostream OS(Result);
- OS << getAssumedNoFPClass();
- return Result;
- }
-
- void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
- SmallVectorImpl<Attribute> &Attrs) const override {
- Attrs.emplace_back(Attribute::getWithNoFPClass(Ctx, getAssumedNoFPClass()));
- }
-};
-
-struct AANoFPClassFloating : public AANoFPClassImpl {
- AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
- : AANoFPClassImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- SmallVector<AA::ValueAndContext> Values;
- bool UsedAssumedInformation = false;
- if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
- AA::AnyScope, UsedAssumedInformation)) {
- Values.push_back({getAssociatedValue(), getCtxI()});
- }
-
- StateType T;
- auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
- const auto *AA = A.getAAFor<AANoFPClass>(*this, IRPosition::value(V),
- DepClassTy::REQUIRED);
- if (!AA || this == AA) {
- T.indicatePessimisticFixpoint();
- } else {
- const AANoFPClass::StateType &S =
- static_cast<const AANoFPClass::StateType &>(AA->getState());
- T ^= S;
- }
- return T.isValidState();
- };
-
- for (const auto &VAC : Values)
- if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
- return indicatePessimisticFixpoint();
-
- return clampStateAndIndicateChange(getState(), T);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FNRET_ATTR(nofpclass)
- }
-};
-
-struct AANoFPClassReturned final
- : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
- AANoFPClassImpl::StateType, false, Attribute::None, false> {
- AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
- : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
- AANoFPClassImpl::StateType, false, Attribute::None, false>(
- IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FNRET_ATTR(nofpclass)
- }
-};
-
-struct AANoFPClassArgument final
- : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
- AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
- : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
-};
-
-struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
- AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AANoFPClassFloating(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSARG_ATTR(nofpclass)
- }
-};
-
-struct AANoFPClassCallSiteReturned final
- : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
- AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSRET_ATTR(nofpclass)
- }
-};
-
-struct AACallEdgesImpl : public AACallEdges {
- AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
-
- const SetVector<Function *> &getOptimisticEdges() const override {
- return CalledFunctions;
- }
-
- bool hasUnknownCallee() const override { return HasUnknownCallee; }
-
- bool hasNonAsmUnknownCallee() const override {
- return HasUnknownCalleeNonAsm;
- }
-
- const std::string getAsStr(Attributor *A) const override {
- return "CallEdges[" + std::to_string(HasUnknownCallee) + "," +
- std::to_string(CalledFunctions.size()) + "]";
- }
-
- void trackStatistics() const override {}
-
-protected:
- void addCalledFunction(Function *Fn, ChangeStatus &Change) {
- if (CalledFunctions.insert(Fn)) {
- Change = ChangeStatus::CHANGED;
- LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
- << "\n");
- }
- }
-
- void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
- if (!HasUnknownCallee)
- Change = ChangeStatus::CHANGED;
- if (NonAsm && !HasUnknownCalleeNonAsm)
- Change = ChangeStatus::CHANGED;
- HasUnknownCalleeNonAsm |= NonAsm;
- HasUnknownCallee = true;
- }
-
-private:
- /// Optimistic set of functions that might be called by this position.
- SetVector<Function *> CalledFunctions;
-
- /// Is there any call with a unknown callee.
- bool HasUnknownCallee = false;
-
- /// Is there any call with a unknown callee, excluding any inline asm.
- bool HasUnknownCalleeNonAsm = false;
-};
-
-struct AACallEdgesCallSite : public AACallEdgesImpl {
- AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
- : AACallEdgesImpl(IRP, A) {}
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- ChangeStatus Change = ChangeStatus::UNCHANGED;
-
- auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
- if (Function *Fn = dyn_cast<Function>(&V)) {
- addCalledFunction(Fn, Change);
- } else {
- LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
- setHasUnknownCallee(true, Change);
- }
-
- // Explore all values.
- return true;
- };
-
- SmallVector<AA::ValueAndContext> Values;
- // Process any value that we might call.
- auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
- if (isa<Constant>(V)) {
- VisitValue(*V, CtxI);
- return;
- }
-
- bool UsedAssumedInformation = false;
- Values.clear();
- if (!A.getAssumedSimplifiedValues(IRPosition::value(*V), *this, Values,
- AA::AnyScope, UsedAssumedInformation)) {
- Values.push_back({*V, CtxI});
- }
- for (auto &VAC : Values)
- VisitValue(*VAC.getValue(), VAC.getCtxI());
- };
-
- CallBase *CB = cast<CallBase>(getCtxI());
-
- if (auto *IA = dyn_cast<InlineAsm>(CB->getCalledOperand())) {
- if (IA->hasSideEffects() &&
- !hasAssumption(*CB->getCaller(), "ompx_no_call_asm") &&
- !hasAssumption(*CB, "ompx_no_call_asm")) {
- setHasUnknownCallee(false, Change);
- }
- return Change;
- }
-
- if (CB->isIndirectCall())
- if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
- *this, getIRPosition(), DepClassTy::OPTIONAL))
- if (IndirectCallAA->foreachCallee(
- [&](Function *Fn) { return VisitValue(*Fn, CB); }))
- return Change;
-
- // The most simple case.
- ProcessCalledOperand(CB->getCalledOperand(), CB);
-
- // Process callback functions.
- SmallVector<const Use *, 4u> CallbackUses;
- AbstractCallSite::getCallbackUses(*CB, CallbackUses);
- for (const Use *U : CallbackUses)
- ProcessCalledOperand(U->get(), CB);
-
- return Change;
- }
-};
-
-struct AACallEdgesFunction : public AACallEdgesImpl {
- AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
- : AACallEdgesImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- ChangeStatus Change = ChangeStatus::UNCHANGED;
-
- auto ProcessCallInst = [&](Instruction &Inst) {
- CallBase &CB = cast<CallBase>(Inst);
-
- auto *CBEdges = A.getAAFor<AACallEdges>(
- *this, IRPosition::callsite_function(CB), DepClassTy::REQUIRED);
- if (!CBEdges)
- return false;
- if (CBEdges->hasNonAsmUnknownCallee())
- setHasUnknownCallee(true, Change);
- if (CBEdges->hasUnknownCallee())
- setHasUnknownCallee(false, Change);
-
- for (Function *F : CBEdges->getOptimisticEdges())
- addCalledFunction(F, Change);
-
- return true;
- };
-
- // Visit all callable instructions.
- bool UsedAssumedInformation = false;
- if (!A.checkForAllCallLikeInstructions(ProcessCallInst, *this,
- UsedAssumedInformation,
- /* CheckBBLivenessOnly */ true)) {
- // If we haven't looked at all call like instructions, assume that there
- // are unknown callees.
- setHasUnknownCallee(true, Change);
- }
-
- return Change;
- }
-};
-
-/// -------------------AAInterFnReachability Attribute--------------------------
-
-struct AAInterFnReachabilityFunction
- : public CachedReachabilityAA<AAInterFnReachability, Function> {
- using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
- AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- bool instructionCanReach(
- Attributor &A, const Instruction &From, const Function &To,
- const AA::InstExclusionSetTy *ExclusionSet) const override {
- assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
- auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
-
- RQITy StackRQI(A, From, To, ExclusionSet, false);
- typename RQITy::Reachable Result;
- if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
- return NonConstThis->isReachableImpl(A, StackRQI,
- /*IsTemporaryRQI=*/true);
- return Result == RQITy::Reachable::Yes;
- }
-
- bool isReachableImpl(Attributor &A, RQITy &RQI,
- bool IsTemporaryRQI) override {
- const Instruction *EntryI =
- &RQI.From->getFunction()->getEntryBlock().front();
- if (EntryI != RQI.From &&
- !instructionCanReach(A, *EntryI, *RQI.To, nullptr))
- return rememberResult(A, RQITy::Reachable::No, RQI, false,
- IsTemporaryRQI);
-
- auto CheckReachableCallBase = [&](CallBase *CB) {
- auto *CBEdges = A.getAAFor<AACallEdges>(
- *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);
- if (!CBEdges || !CBEdges->getState().isValidState())
- return false;
- // TODO Check To backwards in this case.
- if (CBEdges->hasUnknownCallee())
- return false;
-
- for (Function *Fn : CBEdges->getOptimisticEdges()) {
- if (Fn == RQI.To)
- return false;
-
- if (Fn->isDeclaration()) {
- if (Fn->hasFnAttribute(Attribute::NoCallback))
- continue;
- // TODO Check To backwards in this case.
- return false;
- }
-
- if (Fn == getAnchorScope()) {
- if (EntryI == RQI.From)
- continue;
- return false;
- }
-
- const AAInterFnReachability *InterFnReachability =
- A.getAAFor<AAInterFnReachability>(*this, IRPosition::function(*Fn),
- DepClassTy::OPTIONAL);
-
- const Instruction &FnFirstInst = Fn->getEntryBlock().front();
- if (!InterFnReachability ||
- InterFnReachability->instructionCanReach(A, FnFirstInst, *RQI.To,
- RQI.ExclusionSet))
- return false;
- }
- return true;
- };
-
- const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
- *this, IRPosition::function(*RQI.From->getFunction()),
- DepClassTy::OPTIONAL);
-
- // Determine call like instructions that we can reach from the inst.
- auto CheckCallBase = [&](Instruction &CBInst) {
- // There are usually less nodes in the call graph, check inter function
- // reachability first.
- if (CheckReachableCallBase(cast<CallBase>(&CBInst)))
- return true;
- return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
- A, *RQI.From, CBInst, RQI.ExclusionSet);
- };
-
- bool UsedExclusionSet = /* conservative */ true;
- bool UsedAssumedInformation = false;
- if (!A.checkForAllCallLikeInstructions(CheckCallBase, *this,
- UsedAssumedInformation,
- /* CheckBBLivenessOnly */ true))
- return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
- IsTemporaryRQI);
-
- return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
- IsTemporaryRQI);
- }
-
- void trackStatistics() const override {}
-};
-} // namespace
-
-template <typename AAType>
-static std::optional<Constant *>
-askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,
- const IRPosition &IRP, Type &Ty) {
- if (!Ty.isIntegerTy())
- return nullptr;
-
- // This will also pass the call base context.
- const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
- if (!AA)
- return nullptr;
-
- std::optional<Constant *> COpt = AA->getAssumedConstant(A);
-
- if (!COpt.has_value()) {
- A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);
- return std::nullopt;
- }
- if (auto *C = *COpt) {
- A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);
- return C;
- }
- return nullptr;
-}
-
-Value *AAPotentialValues::getSingleValue(
- Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
- SmallVectorImpl<AA::ValueAndContext> &Values) {
- Type &Ty = *IRP.getAssociatedType();
- std::optional<Value *> V;
- for (auto &It : Values) {
- V = AA::combineOptionalValuesInAAValueLatice(V, It.getValue(), &Ty);
- if (V.has_value() && !*V)
- break;
- }
- if (!V.has_value())
- return UndefValue::get(&Ty);
- return *V;
-}
-
-namespace {
-struct AAPotentialValuesImpl : AAPotentialValues {
- using StateType = PotentialLLVMValuesState;
-
- AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
- : AAPotentialValues(IRP, A) {}
-
- /// See AbstractAttribute::initialize(..).
- void initialize(Attributor &A) override {
- if (A.hasSimplificationCallback(getIRPosition())) {
- indicatePessimisticFixpoint();
- return;
- }
- Value *Stripped = getAssociatedValue().stripPointerCasts();
- auto *CE = dyn_cast<ConstantExpr>(Stripped);
- if (isa<Constant>(Stripped) &&
- (!CE || CE->getOpcode() != Instruction::ICmp)) {
- addValue(A, getState(), *Stripped, getCtxI(), AA::AnyScope,
- getAnchorScope());
- indicateOptimisticFixpoint();
- return;
- }
- AAPotentialValues::initialize(A);
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- std::string Str;
- llvm::raw_string_ostream OS(Str);
- OS << getState();
- return OS.str();
- }
-
- template <typename AAType>
- static std::optional<Value *> askOtherAA(Attributor &A,
- const AbstractAttribute &AA,
- const IRPosition &IRP, Type &Ty) {
- if (isa<Constant>(IRP.getAssociatedValue()))
- return &IRP.getAssociatedValue();
- std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
- if (!C)
- return std::nullopt;
- if (*C)
- if (auto *CC = AA::getWithType(**C, Ty))
- return CC;
- return nullptr;
- }
-
- virtual void addValue(Attributor &A, StateType &State, Value &V,
- const Instruction *CtxI, AA::ValueScope S,
- Function *AnchorScope) const {
-
- IRPosition ValIRP = IRPosition::value(V);
- if (auto *CB = dyn_cast_or_null<CallBase>(CtxI)) {
- for (const auto &U : CB->args()) {
- if (U.get() != &V)
- continue;
- ValIRP = IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U));
- break;
- }
- }
-
- Value *VPtr = &V;
- if (ValIRP.getAssociatedType()->isIntegerTy()) {
- Type &Ty = *getAssociatedType();
- std::optional<Value *> SimpleV =
- askOtherAA<AAValueConstantRange>(A, *this, ValIRP, Ty);
- if (SimpleV.has_value() && !*SimpleV) {
- auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
- *this, ValIRP, DepClassTy::OPTIONAL);
- if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
- for (const auto &It : PotentialConstantsAA->getAssumedSet())
- State.unionAssumed({{*ConstantInt::get(&Ty, It), nullptr}, S});
- if (PotentialConstantsAA->undefIsContained())
- State.unionAssumed({{*UndefValue::get(&Ty), nullptr}, S});
- return;
- }
- }
- if (!SimpleV.has_value())
- return;
-
- if (*SimpleV)
- VPtr = *SimpleV;
- }
-
- if (isa<ConstantInt>(VPtr))
- CtxI = nullptr;
- if (!AA::isValidInScope(*VPtr, AnchorScope))
- S = AA::ValueScope(S | AA::Interprocedural);
-
- State.unionAssumed({{*VPtr, CtxI}, S});
- }
-
- /// Helper struct to tie a value+context pair together with the scope for
- /// which this is the simplified version.
- struct ItemInfo {
- AA::ValueAndContext I;
- AA::ValueScope S;
-
- bool operator==(const ItemInfo &II) const {
- return II.I == I && II.S == S;
- };
- bool operator<(const ItemInfo &II) const {
- if (I == II.I)
- return S < II.S;
- return I < II.I;
- };
- };
-
- bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
- SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
- for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
- if (!(CS & S))
- continue;
-
- bool UsedAssumedInformation = false;
- SmallVector<AA::ValueAndContext> Values;
- if (!A.getAssumedSimplifiedValues(IRP, this, Values, CS,
- UsedAssumedInformation))
- return false;
-
- for (auto &It : Values)
- ValueScopeMap[It] += CS;
- }
- for (auto &It : ValueScopeMap)
- addValue(A, getState(), *It.first.getValue(), It.first.getCtxI(),
- AA::ValueScope(It.second), getAnchorScope());
-
- return true;
- }
-
- void giveUpOnIntraprocedural(Attributor &A) {
- auto NewS = StateType::getBestState(getState());
- for (const auto &It : getAssumedSet()) {
- if (It.second == AA::Intraprocedural)
- continue;
- addValue(A, NewS, *It.first.getValue(), It.first.getCtxI(),
- AA::Interprocedural, getAnchorScope());
- }
- assert(!undefIsContained() && "Undef should be an explicit value!");
- addValue(A, NewS, getAssociatedValue(), getCtxI(), AA::Intraprocedural,
- getAnchorScope());
- getState() = NewS;
- }
-
- /// See AbstractState::indicatePessimisticFixpoint(...).
- ChangeStatus indicatePessimisticFixpoint() override {
- getState() = StateType::getBestState(getState());
- getState().unionAssumed({{getAssociatedValue(), getCtxI()}, AA::AnyScope});
- AAPotentialValues::indicateOptimisticFixpoint();
- return ChangeStatus::CHANGED;
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- return indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- SmallVector<AA::ValueAndContext> Values;
- for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
- Values.clear();
- if (!getAssumedSimplifiedValues(A, Values, S))
- continue;
- Value &OldV = getAssociatedValue();
- if (isa<UndefValue>(OldV))
- continue;
- Value *NewV = getSingleValue(A, *this, getIRPosition(), Values);
- if (!NewV || NewV == &OldV)
- continue;
- if (getCtxI() &&
- !AA::isValidAtPosition({*NewV, *getCtxI()}, A.getInfoCache()))
- continue;
- if (A.changeAfterManifest(getIRPosition(), *NewV))
- return ChangeStatus::CHANGED;
- }
- return ChangeStatus::UNCHANGED;
- }
-
- bool getAssumedSimplifiedValues(
- Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
- AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
- if (!isValidState())
- return false;
- bool UsedAssumedInformation = false;
- for (const auto &It : getAssumedSet())
- if (It.second & S) {
- if (RecurseForSelectAndPHI && (isa<PHINode>(It.first.getValue()) ||
- isa<SelectInst>(It.first.getValue()))) {
- if (A.getAssumedSimplifiedValues(
- IRPosition::inst(*cast<Instruction>(It.first.getValue())),
- this, Values, S, UsedAssumedInformation))
- continue;
- }
- Values.push_back(It.first);
- }
- assert(!undefIsContained() && "Undef should be an explicit value!");
- return true;
- }
-};
-
-struct AAPotentialValuesFloating : AAPotentialValuesImpl {
- AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
- : AAPotentialValuesImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto AssumedBefore = getAssumed();
-
- genericValueTraversal(A, &getAssociatedValue());
-
- return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- /// Helper struct to remember which AAIsDead instances we actually used.
- struct LivenessInfo {
- const AAIsDead *LivenessAA = nullptr;
- bool AnyDead = false;
- };
-
- /// Check if \p Cmp is a comparison we can simplify.
- ///
- /// We handle multiple cases, one in which at least one operand is an
- /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
- /// operand. Return true if successful, in that case Worklist will be updated.
- bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
- CmpInst::Predicate Pred, ItemInfo II,
- SmallVectorImpl<ItemInfo> &Worklist) {
-
- // Simplify the operands first.
- bool UsedAssumedInformation = false;
- SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
- auto GetSimplifiedValues = [&](Value &V,
- SmallVector<AA::ValueAndContext> &Values) {
- if (!A.getAssumedSimplifiedValues(
- IRPosition::value(V, getCallBaseContext()), this, Values,
- AA::Intraprocedural, UsedAssumedInformation)) {
- Values.clear();
- Values.push_back(AA::ValueAndContext{V, II.I.getCtxI()});
- }
- return Values.empty();
- };
- if (GetSimplifiedValues(*LHS, LHSValues))
- return true;
- if (GetSimplifiedValues(*RHS, RHSValues))
- return true;
-
- LLVMContext &Ctx = LHS->getContext();
-
- InformationCache &InfoCache = A.getInfoCache();
- Instruction *CmpI = dyn_cast<Instruction>(&Cmp);
- Function *F = CmpI ? CmpI->getFunction() : nullptr;
- const auto *DT =
- F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F)
- : nullptr;
- const auto *TLI =
- F ? A.getInfoCache().getTargetLibraryInfoForFunction(*F) : nullptr;
- auto *AC =
- F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F)
- : nullptr;
-
- const DataLayout &DL = A.getDataLayout();
- SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
-
- auto CheckPair = [&](Value &LHSV, Value &RHSV) {
- if (isa<UndefValue>(LHSV) || isa<UndefValue>(RHSV)) {
- addValue(A, getState(), *UndefValue::get(Cmp.getType()),
- /* CtxI */ nullptr, II.S, getAnchorScope());
- return true;
- }
-
- // Handle the trivial case first in which we don't even need to think
- // about null or non-null.
- if (&LHSV == &RHSV &&
- (CmpInst::isTrueWhenEqual(Pred) || CmpInst::isFalseWhenEqual(Pred))) {
- Constant *NewV = ConstantInt::get(Type::getInt1Ty(Ctx),
- CmpInst::isTrueWhenEqual(Pred));
- addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
- getAnchorScope());
- return true;
- }
-
- auto *TypedLHS = AA::getWithType(LHSV, *LHS->getType());
- auto *TypedRHS = AA::getWithType(RHSV, *RHS->getType());
- if (TypedLHS && TypedRHS) {
- Value *NewV = simplifyCmpInst(Pred, TypedLHS, TypedRHS, Q);
- if (NewV && NewV != &Cmp) {
- addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
- getAnchorScope());
- return true;
- }
- }
-
- // From now on we only handle equalities (==, !=).
- if (!CmpInst::isEquality(Pred))
- return false;
-
- bool LHSIsNull = isa<ConstantPointerNull>(LHSV);
- bool RHSIsNull = isa<ConstantPointerNull>(RHSV);
- if (!LHSIsNull && !RHSIsNull)
- return false;
-
- // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
- // non-nullptr operand and if we assume it's non-null we can conclude the
- // result of the comparison.
- assert((LHSIsNull || RHSIsNull) &&
- "Expected nullptr versus non-nullptr comparison at this point");
-
- // The index is the operand that we assume is not null.
- unsigned PtrIdx = LHSIsNull;
- bool IsKnownNonNull;
- bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
- A, this, IRPosition::value(*(PtrIdx ? &RHSV : &LHSV)),
- DepClassTy::REQUIRED, IsKnownNonNull);
- if (!IsAssumedNonNull)
- return false;
-
- // The new value depends on the predicate, true for != and false for ==.
- Constant *NewV =
- ConstantInt::get(Type::getInt1Ty(Ctx), Pred == CmpInst::ICMP_NE);
- addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
- getAnchorScope());
- return true;
- };
-
- for (auto &LHSValue : LHSValues)
- for (auto &RHSValue : RHSValues)
- if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
- return false;
- return true;
- }
-
- bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
- SmallVectorImpl<ItemInfo> &Worklist) {
- const Instruction *CtxI = II.I.getCtxI();
- bool UsedAssumedInformation = false;
-
- std::optional<Constant *> C =
- A.getAssumedConstant(*SI.getCondition(), *this, UsedAssumedInformation);
- bool NoValueYet = !C.has_value();
- if (NoValueYet || isa_and_nonnull<UndefValue>(*C))
- return true;
- if (auto *CI = dyn_cast_or_null<ConstantInt>(*C)) {
- if (CI->isZero())
- Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});
- else
- Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});
- } else if (&SI == &getAssociatedValue()) {
- // We could not simplify the condition, assume both values.
- Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});
- Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});
- } else {
- std::optional<Value *> SimpleV = A.getAssumedSimplified(
- IRPosition::inst(SI), *this, UsedAssumedInformation, II.S);
- if (!SimpleV.has_value())
- return true;
- if (*SimpleV) {
- addValue(A, getState(), **SimpleV, CtxI, II.S, getAnchorScope());
- return true;
- }
- return false;
- }
- return true;
- }
-
- bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
- SmallVectorImpl<ItemInfo> &Worklist) {
- SmallSetVector<Value *, 4> PotentialCopies;
- SmallSetVector<Instruction *, 4> PotentialValueOrigins;
- bool UsedAssumedInformation = false;
- if (!AA::getPotentiallyLoadedValues(A, LI, PotentialCopies,
- PotentialValueOrigins, *this,
- UsedAssumedInformation,
- /* OnlyExact */ true)) {
- LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
- "loaded values for load instruction "
- << LI << "\n");
- return false;
- }
-
- // Do not simplify loads that are only used in llvm.assume if we cannot also
- // remove all stores that may feed into the load. The reason is that the
- // assume is probably worth something as long as the stores are around.
- InformationCache &InfoCache = A.getInfoCache();
- if (InfoCache.isOnlyUsedByAssume(LI)) {
- if (!llvm::all_of(PotentialValueOrigins, [&](Instruction *I) {
- if (!I || isa<AssumeInst>(I))
- return true;
- if (auto *SI = dyn_cast<StoreInst>(I))
- return A.isAssumedDead(SI->getOperandUse(0), this,
- /* LivenessAA */ nullptr,
- UsedAssumedInformation,
- /* CheckBBLivenessOnly */ false);
- return A.isAssumedDead(*I, this, /* LivenessAA */ nullptr,
- UsedAssumedInformation,
- /* CheckBBLivenessOnly */ false);
- })) {
- LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
- "and we cannot delete all the stores: "
- << LI << "\n");
- return false;
- }
- }
-
- // Values have to be dynamically unique or we loose the fact that a
- // single llvm::Value might represent two runtime values (e.g.,
- // stack locations in different recursive calls).
- const Instruction *CtxI = II.I.getCtxI();
- bool ScopeIsLocal = (II.S & AA::Intraprocedural);
- bool AllLocal = ScopeIsLocal;
- bool DynamicallyUnique = llvm::all_of(PotentialCopies, [&](Value *PC) {
- AllLocal &= AA::isValidInScope(*PC, getAnchorScope());
- return AA::isDynamicallyUnique(A, *this, *PC);
- });
- if (!DynamicallyUnique) {
- LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
- "values are dynamically unique: "
- << LI << "\n");
- return false;
- }
-
- for (auto *PotentialCopy : PotentialCopies) {
- if (AllLocal) {
- Worklist.push_back({{*PotentialCopy, CtxI}, II.S});
- } else {
- Worklist.push_back({{*PotentialCopy, CtxI}, AA::Interprocedural});
- }
- }
- if (!AllLocal && ScopeIsLocal)
- addValue(A, getState(), LI, CtxI, AA::Intraprocedural, getAnchorScope());
- return true;
- }
-
- bool handlePHINode(
- Attributor &A, PHINode &PHI, ItemInfo II,
- SmallVectorImpl<ItemInfo> &Worklist,
- SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
- auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
- LivenessInfo &LI = LivenessAAs[&F];
- if (!LI.LivenessAA)
- LI.LivenessAA = A.getAAFor<AAIsDead>(*this, IRPosition::function(F),
- DepClassTy::NONE);
- return LI;
- };
-
- if (&PHI == &getAssociatedValue()) {
- LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
- const auto *CI =
- A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
- *PHI.getFunction());
-
- Cycle *C = nullptr;
- bool CyclePHI = mayBeInCycle(CI, &PHI, /* HeaderOnly */ true, &C);
- for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
- BasicBlock *IncomingBB = PHI.getIncomingBlock(u);
- if (LI.LivenessAA &&
- LI.LivenessAA->isEdgeDead(IncomingBB, PHI.getParent())) {
- LI.AnyDead = true;
- continue;
- }
- Value *V = PHI.getIncomingValue(u);
- if (V == &PHI)
- continue;
-
- // If the incoming value is not the PHI but an instruction in the same
- // cycle we might have multiple versions of it flying around.
- if (CyclePHI && isa<Instruction>(V) &&
- (!C || C->contains(cast<Instruction>(V)->getParent())))
- return false;
-
- Worklist.push_back({{*V, IncomingBB->getTerminator()}, II.S});
- }
- return true;
- }
-
- bool UsedAssumedInformation = false;
- std::optional<Value *> SimpleV = A.getAssumedSimplified(
- IRPosition::inst(PHI), *this, UsedAssumedInformation, II.S);
- if (!SimpleV.has_value())
- return true;
- if (!(*SimpleV))
- return false;
- addValue(A, getState(), **SimpleV, &PHI, II.S, getAnchorScope());
- return true;
- }
-
- /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
- /// simplify any operand of the instruction \p I. Return true if successful,
- /// in that case Worklist will be updated.
- bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
- SmallVectorImpl<ItemInfo> &Worklist) {
- bool SomeSimplified = false;
- bool UsedAssumedInformation = false;
-
- SmallVector<Value *, 8> NewOps(I.getNumOperands());
- int Idx = 0;
- for (Value *Op : I.operands()) {
- const auto &SimplifiedOp = A.getAssumedSimplified(
- IRPosition::value(*Op, getCallBaseContext()), *this,
- UsedAssumedInformation, AA::Intraprocedural);
- // If we are not sure about any operand we are not sure about the entire
- // instruction, we'll wait.
- if (!SimplifiedOp.has_value())
- return true;
-
- if (*SimplifiedOp)
- NewOps[Idx] = *SimplifiedOp;
- else
- NewOps[Idx] = Op;
-
- SomeSimplified |= (NewOps[Idx] != Op);
- ++Idx;
- }
-
- // We won't bother with the InstSimplify interface if we didn't simplify any
- // operand ourselves.
- if (!SomeSimplified)
- return false;
-
- InformationCache &InfoCache = A.getInfoCache();
- Function *F = I.getFunction();
- const auto *DT =
- InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);
- const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
- auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
-
- const DataLayout &DL = I.getModule()->getDataLayout();
- SimplifyQuery Q(DL, TLI, DT, AC, &I);
- Value *NewV = simplifyInstructionWithOperands(&I, NewOps, Q);
- if (!NewV || NewV == &I)
- return false;
-
- LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
- << *NewV << "\n");
- Worklist.push_back({{*NewV, II.I.getCtxI()}, II.S});
- return true;
- }
-
- bool simplifyInstruction(
- Attributor &A, Instruction &I, ItemInfo II,
- SmallVectorImpl<ItemInfo> &Worklist,
- SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
- if (auto *CI = dyn_cast<CmpInst>(&I))
- return handleCmp(A, *CI, CI->getOperand(0), CI->getOperand(1),
- CI->getPredicate(), II, Worklist);
-
- switch (I.getOpcode()) {
- case Instruction::Select:
- return handleSelectInst(A, cast<SelectInst>(I), II, Worklist);
- case Instruction::PHI:
- return handlePHINode(A, cast<PHINode>(I), II, Worklist, LivenessAAs);
- case Instruction::Load:
- return handleLoadInst(A, cast<LoadInst>(I), II, Worklist);
- default:
- return handleGenericInst(A, I, II, Worklist);
- };
- return false;
- }
-
- void genericValueTraversal(Attributor &A, Value *InitialV) {
- SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
-
- SmallSet<ItemInfo, 16> Visited;
- SmallVector<ItemInfo, 16> Worklist;
- Worklist.push_back({{*InitialV, getCtxI()}, AA::AnyScope});
-
- int Iteration = 0;
- do {
- ItemInfo II = Worklist.pop_back_val();
- Value *V = II.I.getValue();
- assert(V);
- const Instruction *CtxI = II.I.getCtxI();
- AA::ValueScope S = II.S;
-
- // Check if we should process the current value. To prevent endless
- // recursion keep a record of the values we followed!
- if (!Visited.insert(II).second)
- continue;
-
- // Make sure we limit the compile time for complex expressions.
- if (Iteration++ >= MaxPotentialValuesIterations) {
- LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
- << Iteration << "!\n");
- addValue(A, getState(), *V, CtxI, S, getAnchorScope());
- continue;
- }
-
- // Explicitly look through calls with a "returned" attribute if we do
- // not have a pointer as stripPointerCasts only works on them.
- Value *NewV = nullptr;
- if (V->getType()->isPointerTy()) {
- NewV = AA::getWithType(*V->stripPointerCasts(), *V->getType());
- } else {
- if (auto *CB = dyn_cast<CallBase>(V))
- if (auto *Callee =
- dyn_cast_if_present<Function>(CB->getCalledOperand())) {
- for (Argument &Arg : Callee->args())
- if (Arg.hasReturnedAttr()) {
- NewV = CB->getArgOperand(Arg.getArgNo());
- break;
- }
- }
- }
- if (NewV && NewV != V) {
- Worklist.push_back({{*NewV, CtxI}, S});
- continue;
- }
-
- if (auto *CE = dyn_cast<ConstantExpr>(V)) {
- if (CE->getOpcode() == Instruction::ICmp)
- if (handleCmp(A, *CE, CE->getOperand(0), CE->getOperand(1),
- CmpInst::Predicate(CE->getPredicate()), II, Worklist))
- continue;
- }
-
- if (auto *I = dyn_cast<Instruction>(V)) {
- if (simplifyInstruction(A, *I, II, Worklist, LivenessAAs))
- continue;
- }
-
- if (V != InitialV || isa<Argument>(V))
- if (recurseForValue(A, IRPosition::value(*V), II.S))
- continue;
-
- // If we haven't stripped anything we give up.
- if (V == InitialV && CtxI == getCtxI()) {
- indicatePessimisticFixpoint();
- return;
- }
-
- addValue(A, getState(), *V, CtxI, S, getAnchorScope());
- } while (!Worklist.empty());
-
- // If we actually used liveness information so we have to record a
- // dependence.
- for (auto &It : LivenessAAs)
- if (It.second.AnyDead)
- A.recordDependence(*It.second.LivenessAA, *this, DepClassTy::OPTIONAL);
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(potential_values)
- }
-};
-
-struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
- using Base = AAPotentialValuesImpl;
- AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- /// See AbstractAttribute::initialize(..).
- void initialize(Attributor &A) override {
- auto &Arg = cast<Argument>(getAssociatedValue());
- if (Arg.hasPointeeInMemoryValueAttr())
- indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto AssumedBefore = getAssumed();
-
- unsigned ArgNo = getCalleeArgNo();
-
- bool UsedAssumedInformation = false;
- SmallVector<AA::ValueAndContext> Values;
- auto CallSitePred = [&](AbstractCallSite ACS) {
- const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
- if (CSArgIRP.getPositionKind() == IRP_INVALID)
- return false;
-
- if (!A.getAssumedSimplifiedValues(CSArgIRP, this, Values,
- AA::Interprocedural,
- UsedAssumedInformation))
- return false;
-
- return isValidState();
- };
-
- if (!A.checkForAllCallSites(CallSitePred, *this,
- /* RequireAllCallSites */ true,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- Function *Fn = getAssociatedFunction();
- bool AnyNonLocal = false;
- for (auto &It : Values) {
- if (isa<Constant>(It.getValue())) {
- addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,
- getAnchorScope());
- continue;
- }
- if (!AA::isDynamicallyUnique(A, *this, *It.getValue()))
- return indicatePessimisticFixpoint();
-
- if (auto *Arg = dyn_cast<Argument>(It.getValue()))
- if (Arg->getParent() == Fn) {
- addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,
- getAnchorScope());
- continue;
- }
- addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::Interprocedural,
- getAnchorScope());
- AnyNonLocal = true;
- }
- assert(!undefIsContained() && "Undef should be an explicit value!");
- if (AnyNonLocal)
- giveUpOnIntraprocedural(A);
-
- return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_ARG_ATTR(potential_values)
- }
-};
-
-struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
- using Base = AAPotentialValuesFloating;
- AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
- : Base(IRP, A) {}
-
- /// See AbstractAttribute::initialize(..).
- void initialize(Attributor &A) override {
- Function *F = getAssociatedFunction();
- if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
- indicatePessimisticFixpoint();
- return;
- }
-
- for (Argument &Arg : F->args())
- if (Arg.hasReturnedAttr()) {
- addValue(A, getState(), Arg, nullptr, AA::AnyScope, F);
- ReturnedArg = &Arg;
- break;
- }
- if (!A.isFunctionIPOAmendable(*F) ||
- A.hasSimplificationCallback(getIRPosition())) {
- if (!ReturnedArg)
- indicatePessimisticFixpoint();
- else
- indicateOptimisticFixpoint();
- }
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto AssumedBefore = getAssumed();
- bool UsedAssumedInformation = false;
-
- SmallVector<AA::ValueAndContext> Values;
- Function *AnchorScope = getAnchorScope();
- auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
- bool AddValues) {
- for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
- Values.clear();
- if (!A.getAssumedSimplifiedValues(IRPosition::value(V), this, Values, S,
- UsedAssumedInformation,
- /* RecurseForSelectAndPHI */ true))
- return false;
- if (!AddValues)
- continue;
- for (const AA::ValueAndContext &VAC : Values)
- addValue(A, getState(), *VAC.getValue(),
- VAC.getCtxI() ? VAC.getCtxI() : CtxI, S, AnchorScope);
- }
- return true;
- };
-
- if (ReturnedArg) {
- HandleReturnedValue(*ReturnedArg, nullptr, true);
- } else {
- auto RetInstPred = [&](Instruction &RetI) {
- bool AddValues = true;
- if (isa<PHINode>(RetI.getOperand(0)) ||
- isa<SelectInst>(RetI.getOperand(0))) {
- addValue(A, getState(), *RetI.getOperand(0), &RetI, AA::AnyScope,
- AnchorScope);
- AddValues = false;
- }
- return HandleReturnedValue(*RetI.getOperand(0), &RetI, AddValues);
- };
-
- if (!A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
- UsedAssumedInformation,
- /* CheckBBLivenessOnly */ true))
- return indicatePessimisticFixpoint();
- }
-
- return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- void addValue(Attributor &A, StateType &State, Value &V,
- const Instruction *CtxI, AA::ValueScope S,
- Function *AnchorScope) const override {
- Function *F = getAssociatedFunction();
- if (auto *CB = dyn_cast<CallBase>(&V))
- if (CB->getCalledOperand() == F)
- return;
- Base::addValue(A, State, V, CtxI, S, AnchorScope);
- }
-
- ChangeStatus manifest(Attributor &A) override {
- if (ReturnedArg)
- return ChangeStatus::UNCHANGED;
- SmallVector<AA::ValueAndContext> Values;
- if (!getAssumedSimplifiedValues(A, Values, AA::ValueScope::Intraprocedural,
- /* RecurseForSelectAndPHI */ true))
- return ChangeStatus::UNCHANGED;
- Value *NewVal = getSingleValue(A, *this, getIRPosition(), Values);
- if (!NewVal)
- return ChangeStatus::UNCHANGED;
-
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- if (auto *Arg = dyn_cast<Argument>(NewVal)) {
- STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
- "Number of function with unique return");
- Changed |= A.manifestAttrs(
- IRPosition::argument(*Arg),
- {Attribute::get(Arg->getContext(), Attribute::Returned)});
- STATS_DECLTRACK_ARG_ATTR(returned);
- }
-
- auto RetInstPred = [&](Instruction &RetI) {
- Value *RetOp = RetI.getOperand(0);
- if (isa<UndefValue>(RetOp) || RetOp == NewVal)
- return true;
- if (AA::isValidAtPosition({*NewVal, RetI}, A.getInfoCache()))
- if (A.changeUseAfterManifest(RetI.getOperandUse(0), *NewVal))
- Changed = ChangeStatus::CHANGED;
- return true;
- };
- bool UsedAssumedInformation = false;
- (void)A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
- UsedAssumedInformation,
- /* CheckBBLivenessOnly */ true);
- return Changed;
- }
-
- ChangeStatus indicatePessimisticFixpoint() override {
- return AAPotentialValues::indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override{
- STATS_DECLTRACK_FNRET_ATTR(potential_values)}
-
- /// The argumented with an existing `returned` attribute.
- Argument *ReturnedArg = nullptr;
-};
-
-struct AAPotentialValuesFunction : AAPotentialValuesImpl {
- AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
- : AAPotentialValuesImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
- "not be called");
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_FN_ATTR(potential_values)
- }
-};
-
-struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
- AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
- : AAPotentialValuesFunction(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CS_ATTR(potential_values)
- }
-};
-
-struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
- AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AAPotentialValuesImpl(IRP, A) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto AssumedBefore = getAssumed();
-
- Function *Callee = getAssociatedFunction();
- if (!Callee)
- return indicatePessimisticFixpoint();
-
- bool UsedAssumedInformation = false;
- auto *CB = cast<CallBase>(getCtxI());
- if (CB->isMustTailCall() &&
- !A.isAssumedDead(IRPosition::inst(*CB), this, nullptr,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- SmallVector<AA::ValueAndContext> Values;
- if (!A.getAssumedSimplifiedValues(IRPosition::returned(*Callee), this,
- Values, AA::Intraprocedural,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- Function *Caller = CB->getCaller();
-
- bool AnyNonLocal = false;
- for (auto &It : Values) {
- Value *V = It.getValue();
- std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
- V, *CB, *this, UsedAssumedInformation);
- if (!CallerV.has_value()) {
- // Nothing to do as long as no value was determined.
- continue;
- }
- V = *CallerV ? *CallerV : V;
- if (AA::isDynamicallyUnique(A, *this, *V) &&
- AA::isValidInScope(*V, Caller)) {
- if (*CallerV) {
- SmallVector<AA::ValueAndContext> ArgValues;
- IRPosition IRP = IRPosition::value(*V);
- if (auto *Arg = dyn_cast<Argument>(V))
- if (Arg->getParent() == CB->getCalledOperand())
- IRP = IRPosition::callsite_argument(*CB, Arg->getArgNo());
- if (recurseForValue(A, IRP, AA::AnyScope))
- continue;
- }
- addValue(A, getState(), *V, CB, AA::AnyScope, getAnchorScope());
- } else {
- AnyNonLocal = true;
- break;
- }
- }
- if (AnyNonLocal) {
- Values.clear();
- if (!A.getAssumedSimplifiedValues(IRPosition::returned(*Callee), this,
- Values, AA::Interprocedural,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
- AnyNonLocal = false;
- getState() = PotentialLLVMValuesState::getBestState();
- for (auto &It : Values) {
- Value *V = It.getValue();
- if (!AA::isDynamicallyUnique(A, *this, *V))
- return indicatePessimisticFixpoint();
- if (AA::isValidInScope(*V, Caller)) {
- addValue(A, getState(), *V, CB, AA::AnyScope, getAnchorScope());
- } else {
- AnyNonLocal = true;
- addValue(A, getState(), *V, CB, AA::Interprocedural,
- getAnchorScope());
- }
- }
- if (AnyNonLocal)
- giveUpOnIntraprocedural(A);
- }
- return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- ChangeStatus indicatePessimisticFixpoint() override {
- return AAPotentialValues::indicatePessimisticFixpoint();
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSRET_ATTR(potential_values)
- }
-};
-
-struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
- AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAPotentialValuesFloating(IRP, A) {}
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {
- STATS_DECLTRACK_CSARG_ATTR(potential_values)
- }
-};
-} // namespace
-
-/// ---------------------- Assumption Propagation ------------------------------
-namespace {
-struct AAAssumptionInfoImpl : public AAAssumptionInfo {
- AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
- const DenseSet<StringRef> &Known)
- : AAAssumptionInfo(IRP, A, Known) {}
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- // Don't manifest a universal set if it somehow made it here.
- if (getKnown().isUniversal())
- return ChangeStatus::UNCHANGED;
-
- const IRPosition &IRP = getIRPosition();
- return A.manifestAttrs(
- IRP,
- Attribute::get(IRP.getAnchorValue().getContext(), AssumptionAttrKey,
- llvm::join(getAssumed().getSet(), ",")),
- /* ForceReplace */ true);
- }
-
- bool hasAssumption(const StringRef Assumption) const override {
- return isValidState() && setContains(Assumption);
- }
-
- /// See AbstractAttribute::getAsStr()
- const std::string getAsStr(Attributor *A) const override {
- const SetContents &Known = getKnown();
- const SetContents &Assumed = getAssumed();
-
- const std::string KnownStr =
- llvm::join(Known.getSet().begin(), Known.getSet().end(), ",");
- const std::string AssumedStr =
- (Assumed.isUniversal())
- ? "Universal"
- : llvm::join(Assumed.getSet().begin(), Assumed.getSet().end(), ",");
-
- return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
- }
-};
-
-/// Propagates assumption information from parent functions to all of their
-/// successors. An assumption can be propagated if the containing function
-/// dominates the called function.
-///
-/// We start with a "known" set of assumptions already valid for the associated
-/// function and an "assumed" set that initially contains all possible
-/// assumptions. The assumed set is inter-procedurally updated by narrowing its
-/// contents as concrete values are known. The concrete values are seeded by the
-/// first nodes that are either entries into the call graph, or contains no
-/// assumptions. Each node is updated as the intersection of the assumed state
-/// with all of its predecessors.
-struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
- AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
- : AAAssumptionInfoImpl(IRP, A,
- getAssumptions(*IRP.getAssociatedFunction())) {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- bool Changed = false;
-
- auto CallSitePred = [&](AbstractCallSite ACS) {
- const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
- *this, IRPosition::callsite_function(*ACS.getInstruction()),
- DepClassTy::REQUIRED);
- if (!AssumptionAA)
- return false;
- // Get the set of assumptions shared by all of this function's callers.
- Changed |= getIntersection(AssumptionAA->getAssumed());
- return !getAssumed().empty() || !getKnown().empty();
- };
-
- bool UsedAssumedInformation = false;
- // Get the intersection of all assumptions held by this node's predecessors.
- // If we don't know all the call sites then this is either an entry into the
- // call graph or an empty node. This node is known to only contain its own
- // assumptions and can be propagated to its successors.
- if (!A.checkForAllCallSites(CallSitePred, *this, true,
- UsedAssumedInformation))
- return indicatePessimisticFixpoint();
-
- return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
- }
-
- void trackStatistics() const override {}
-};
-
-/// Assumption Info defined for call sites.
-struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
-
- AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
- : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
- A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);
- }
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
- auto *AssumptionAA =
- A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);
- if (!AssumptionAA)
- return indicatePessimisticFixpoint();
- bool Changed = getIntersection(AssumptionAA->getAssumed());
- return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {}
-
-private:
- /// Helper to initialized the known set as all the assumptions this call and
- /// the callee contain.
- DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
- const CallBase &CB = cast<CallBase>(IRP.getAssociatedValue());
- auto Assumptions = getAssumptions(CB);
- if (const Function *F = CB.getCaller())
- set_union(Assumptions, getAssumptions(*F));
- if (Function *F = IRP.getAssociatedFunction())
- set_union(Assumptions, getAssumptions(*F));
- return Assumptions;
- }
-};
-} // namespace
-
-AACallGraphNode *AACallEdgeIterator::operator*() const {
- return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
- A.getOrCreateAAFor<AACallEdges>(IRPosition::function(**I))));
-}
-
-void AttributorCallGraph::print() { llvm::WriteGraph(outs(), this); }
-
-/// ------------------------ UnderlyingObjects ---------------------------------
-
-namespace {
-struct AAUnderlyingObjectsImpl
- : StateWrapper<BooleanState, AAUnderlyingObjects> {
- using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>;
- AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return std::string("UnderlyingObjects ") +
- (isValidState()
- ? (std::string("inter #") +
- std::to_string(InterAssumedUnderlyingObjects.size()) +
- " objs" + std::string(", intra #") +
- std::to_string(IntraAssumedUnderlyingObjects.size()) +
- " objs")
- : "<invalid>");
- }
-
- /// See AbstractAttribute::trackStatistics()
- void trackStatistics() const override {}
-
- /// See AbstractAttribute::updateImpl(...).
- ChangeStatus updateImpl(Attributor &A) override {
- auto &Ptr = getAssociatedValue();
-
- auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
- AA::ValueScope Scope) {
- bool UsedAssumedInformation = false;
- SmallPtrSet<Value *, 8> SeenObjects;
- SmallVector<AA::ValueAndContext> Values;
-
- if (!A.getAssumedSimplifiedValues(IRPosition::value(Ptr), *this, Values,
- Scope, UsedAssumedInformation))
- return UnderlyingObjects.insert(&Ptr);
-
- bool Changed = false;
-
- for (unsigned I = 0; I < Values.size(); ++I) {
- auto &VAC = Values[I];
- auto *Obj = VAC.getValue();
- Value *UO = getUnderlyingObject(Obj);
- if (UO && UO != VAC.getValue() && SeenObjects.insert(UO).second) {
- const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
- *this, IRPosition::value(*UO), DepClassTy::OPTIONAL);
- auto Pred = [&Values](Value &V) {
- Values.emplace_back(V, nullptr);
- return true;
- };
-
- if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
- llvm_unreachable(
- "The forall call should not return false at this position");
-
- continue;
- }
-
- if (isa<SelectInst>(Obj)) {
- Changed |= handleIndirect(A, *Obj, UnderlyingObjects, Scope);
- continue;
- }
- if (auto *PHI = dyn_cast<PHINode>(Obj)) {
- // Explicitly look through PHIs as we do not care about dynamically
- // uniqueness.
- for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
- Changed |= handleIndirect(A, *PHI->getIncomingValue(u),
- UnderlyingObjects, Scope);
- }
- continue;
- }
-
- Changed |= UnderlyingObjects.insert(Obj);
- }
-
- return Changed;
- };
-
- bool Changed = false;
- Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
- Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
-
- return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
- }
-
- bool forallUnderlyingObjects(
- function_ref<bool(Value &)> Pred,
- AA::ValueScope Scope = AA::Interprocedural) const override {
- if (!isValidState())
- return Pred(getAssociatedValue());
-
- auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
- ? IntraAssumedUnderlyingObjects
- : InterAssumedUnderlyingObjects;
- for (Value *Obj : AssumedUnderlyingObjects)
- if (!Pred(*Obj))
- return false;
-
- return true;
- }
-
-private:
- /// Handle the case where the value is not the actual underlying value, such
- /// as a phi node or a select instruction.
- bool handleIndirect(Attributor &A, Value &V,
- SmallSetVector<Value *, 8> &UnderlyingObjects,
- AA::ValueScope Scope) {
- bool Changed = false;
- const auto *AA = A.getAAFor<AAUnderlyingObjects>(
- *this, IRPosition::value(V), DepClassTy::OPTIONAL);
- auto Pred = [&](Value &V) {
- Changed |= UnderlyingObjects.insert(&V);
- return true;
- };
- if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
- llvm_unreachable(
- "The forall call should not return false at this position");
- return Changed;
- }
-
- /// All the underlying objects collected so far via intra procedural scope.
- SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
- /// All the underlying objects collected so far via inter procedural scope.
- SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
-};
-
-struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
- AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
- : AAUnderlyingObjectsImpl(IRP, A) {}
-};
-
-struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
- AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
- : AAUnderlyingObjectsImpl(IRP, A) {}
-};
-
-struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
- AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
- : AAUnderlyingObjectsImpl(IRP, A) {}
-};
-
-struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
- AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAUnderlyingObjectsImpl(IRP, A) {}
-};
-
-struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
- AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
- : AAUnderlyingObjectsImpl(IRP, A) {}
-};
-
-struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
- AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AAUnderlyingObjectsImpl(IRP, A) {}
-};
-
-struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
- AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
- : AAUnderlyingObjectsImpl(IRP, A) {}
-};
-} // namespace
-
-/// ------------------------ Global Value Info -------------------------------
-namespace {
-struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
- AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
- : AAGlobalValueInfo(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {}
-
- bool checkUse(Attributor &A, const Use &U, bool &Follow,
- SmallVectorImpl<const Value *> &Worklist) {
- Instruction *UInst = dyn_cast<Instruction>(U.getUser());
- if (!UInst) {
- Follow = true;
- return true;
- }
-
- LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
- << *UInst << "\n");
-
- if (auto *Cmp = dyn_cast<ICmpInst>(U.getUser())) {
- int Idx = &Cmp->getOperandUse(0) == &U;
- if (isa<Constant>(Cmp->getOperand(Idx)))
- return true;
- return U == &getAnchorValue();
- }
-
- // Explicitly catch return instructions.
- if (isa<ReturnInst>(UInst)) {
- auto CallSitePred = [&](AbstractCallSite ACS) {
- Worklist.push_back(ACS.getInstruction());
- return true;
- };
- bool UsedAssumedInformation = false;
- // TODO: We should traverse the uses or add a "non-call-site" CB.
- if (!A.checkForAllCallSites(CallSitePred, *UInst->getFunction(),
- /*RequireAllCallSites=*/true, this,
- UsedAssumedInformation))
- return false;
- return true;
- }
-
- // For now we only use special logic for call sites. However, the tracker
- // itself knows about a lot of other non-capturing cases already.
- auto *CB = dyn_cast<CallBase>(UInst);
- if (!CB)
- return false;
- // Direct calls are OK uses.
- if (CB->isCallee(&U))
- return true;
- // Non-argument uses are scary.
- if (!CB->isArgOperand(&U))
- return false;
- // TODO: Iterate callees.
- auto *Fn = dyn_cast<Function>(CB->getCalledOperand());
- if (!Fn || !A.isFunctionIPOAmendable(*Fn))
- return false;
-
- unsigned ArgNo = CB->getArgOperandNo(&U);
- Worklist.push_back(Fn->getArg(ArgNo));
- return true;
- }
-
- ChangeStatus updateImpl(Attributor &A) override {
- unsigned NumUsesBefore = Uses.size();
-
- SmallPtrSet<const Value *, 8> Visited;
- SmallVector<const Value *> Worklist;
- Worklist.push_back(&getAnchorValue());
-
- auto UsePred = [&](const Use &U, bool &Follow) -> bool {
- Uses.insert(&U);
- switch (DetermineUseCaptureKind(U, nullptr)) {
- case UseCaptureKind::NO_CAPTURE:
- return checkUse(A, U, Follow, Worklist);
- case UseCaptureKind::MAY_CAPTURE:
- return checkUse(A, U, Follow, Worklist);
- case UseCaptureKind::PASSTHROUGH:
- Follow = true;
- return true;
- }
- return true;
- };
- auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
- Uses.insert(&OldU);
- return true;
- };
-
- while (!Worklist.empty()) {
- const Value *V = Worklist.pop_back_val();
- if (!Visited.insert(V).second)
- continue;
- if (!A.checkForAllUses(UsePred, *this, *V,
- /* CheckBBLivenessOnly */ true,
- DepClassTy::OPTIONAL,
- /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
- return indicatePessimisticFixpoint();
- }
- }
-
- return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- bool isPotentialUse(const Use &U) const override {
- return !isValidState() || Uses.contains(&U);
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- return ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return "[" + std::to_string(Uses.size()) + " uses]";
- }
-
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
- }
-
-private:
- /// Set of (transitive) uses of this GlobalValue.
- SmallPtrSet<const Use *, 8> Uses;
-};
-} // namespace
-
-/// ------------------------ Indirect Call Info -------------------------------
-namespace {
-struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
- AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
- : AAIndirectCallInfo(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- auto *MD = getCtxI()->getMetadata(LLVMContext::MD_callees);
- if (!MD && !A.isClosedWorldModule())
- return;
-
- if (MD) {
- for (const auto &Op : MD->operands())
- if (Function *Callee = mdconst::dyn_extract_or_null<Function>(Op))
- PotentialCallees.insert(Callee);
- } else if (A.isClosedWorldModule()) {
- ArrayRef<Function *> IndirectlyCallableFunctions =
- A.getInfoCache().getIndirectlyCallableFunctions(A);
- PotentialCallees.insert(IndirectlyCallableFunctions.begin(),
- IndirectlyCallableFunctions.end());
- }
-
- if (PotentialCallees.empty())
- indicateOptimisticFixpoint();
- }
-
- ChangeStatus updateImpl(Attributor &A) override {
- CallBase *CB = cast<CallBase>(getCtxI());
- const Use &CalleeUse = CB->getCalledOperandUse();
- Value *FP = CB->getCalledOperand();
-
- SmallSetVector<Function *, 4> AssumedCalleesNow;
- bool AllCalleesKnownNow = AllCalleesKnown;
-
- auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
- bool &UsedAssumedInformation) {
- const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
- *this, IRPosition::value(PotentialCallee), DepClassTy::OPTIONAL);
- if (!GIAA || GIAA->isPotentialUse(CalleeUse))
- return true;
- UsedAssumedInformation = !GIAA->isAtFixpoint();
- return false;
- };
-
- auto AddPotentialCallees = [&]() {
- for (auto *PotentialCallee : PotentialCallees) {
- bool UsedAssumedInformation = false;
- if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
- AssumedCalleesNow.insert(PotentialCallee);
- }
- };
-
- // Use simplification to find potential callees, if !callees was present,
- // fallback to that set if necessary.
- bool UsedAssumedInformation = false;
- SmallVector<AA::ValueAndContext> Values;
- if (!A.getAssumedSimplifiedValues(IRPosition::value(*FP), this, Values,
- AA::ValueScope::AnyScope,
- UsedAssumedInformation)) {
- if (PotentialCallees.empty())
- return indicatePessimisticFixpoint();
- AddPotentialCallees();
- }
-
- // Try to find a reason for \p Fn not to be a potential callee. If none was
- // found, add it to the assumed callees set.
- auto CheckPotentialCallee = [&](Function &Fn) {
- if (!PotentialCallees.empty() && !PotentialCallees.count(&Fn))
- return false;
-
- auto &CachedResult = FilterResults[&Fn];
- if (CachedResult.has_value())
- return CachedResult.value();
-
- bool UsedAssumedInformation = false;
- if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
- if (!UsedAssumedInformation)
- CachedResult = false;
- return false;
- }
-
- int NumFnArgs = Fn.arg_size();
- int NumCBArgs = CB->arg_size();
-
- // Check if any excess argument (which we fill up with poison) is known to
- // be UB on undef.
- for (int I = NumCBArgs; I < NumFnArgs; ++I) {
- bool IsKnown = false;
- if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
- A, this, IRPosition::argument(*Fn.getArg(I)),
- DepClassTy::OPTIONAL, IsKnown)) {
- if (IsKnown)
- CachedResult = false;
- return false;
- }
- }
-
- CachedResult = true;
- return true;
- };
-
- // Check simplification result, prune known UB callees, also restrict it to
- // the !callees set, if present.
- for (auto &VAC : Values) {
- if (isa<UndefValue>(VAC.getValue()))
- continue;
- if (isa<ConstantPointerNull>(VAC.getValue()) &&
- VAC.getValue()->getType()->getPointerAddressSpace() == 0)
- continue;
- // TODO: Check for known UB, e.g., poison + noundef.
- if (auto *VACFn = dyn_cast<Function>(VAC.getValue())) {
- if (CheckPotentialCallee(*VACFn))
- AssumedCalleesNow.insert(VACFn);
- continue;
- }
- if (!PotentialCallees.empty()) {
- AddPotentialCallees();
- break;
- }
- AllCalleesKnownNow = false;
- }
-
- if (AssumedCalleesNow == AssumedCallees &&
- AllCalleesKnown == AllCalleesKnownNow)
- return ChangeStatus::UNCHANGED;
-
- std::swap(AssumedCallees, AssumedCalleesNow);
- AllCalleesKnown = AllCalleesKnownNow;
- return ChangeStatus::CHANGED;
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- // If we can't specialize at all, give up now.
- if (!AllCalleesKnown && AssumedCallees.empty())
- return ChangeStatus::UNCHANGED;
-
- CallBase *CB = cast<CallBase>(getCtxI());
- bool UsedAssumedInformation = false;
- if (A.isAssumedDead(*CB, this, /*LivenessAA=*/nullptr,
- UsedAssumedInformation))
- return ChangeStatus::UNCHANGED;
-
- ChangeStatus Changed = ChangeStatus::UNCHANGED;
- Value *FP = CB->getCalledOperand();
- if (FP->getType()->getPointerAddressSpace())
- FP = new AddrSpaceCastInst(FP, PointerType::get(FP->getType(), 0),
- FP->getName() + ".as0", CB->getIterator());
-
- bool CBIsVoid = CB->getType()->isVoidTy();
- BasicBlock::iterator IP = CB->getIterator();
- FunctionType *CSFT = CB->getFunctionType();
- SmallVector<Value *> CSArgs(CB->arg_begin(), CB->arg_end());
-
- // If we know all callees and there are none, the call site is (effectively)
- // dead (or UB).
- if (AssumedCallees.empty()) {
- assert(AllCalleesKnown &&
- "Expected all callees to be known if there are none.");
- A.changeToUnreachableAfterManifest(CB);
- return ChangeStatus::CHANGED;
- }
-
- // Special handling for the single callee case.
- if (AllCalleesKnown && AssumedCallees.size() == 1) {
- auto *NewCallee = AssumedCallees.front();
- if (isLegalToPromote(*CB, NewCallee)) {
- promoteCall(*CB, NewCallee, nullptr);
- return ChangeStatus::CHANGED;
- }
- Instruction *NewCall =
- CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,
- CB->getName(), CB->getIterator());
- if (!CBIsVoid)
- A.changeAfterManifest(IRPosition::callsite_returned(*CB), *NewCall);
- A.deleteAfterManifest(*CB);
- return ChangeStatus::CHANGED;
- }
-
- // For each potential value we create a conditional
- //
- // ```
- // if (ptr == value) value(args);
- // else ...
- // ```
- //
- bool SpecializedForAnyCallees = false;
- bool SpecializedForAllCallees = AllCalleesKnown;
- ICmpInst *LastCmp = nullptr;
- SmallVector<Function *, 8> SkippedAssumedCallees;
- SmallVector<std::pair<CallInst *, Instruction *>> NewCalls;
- for (Function *NewCallee : AssumedCallees) {
- if (!A.shouldSpecializeCallSiteForCallee(*this, *CB, *NewCallee)) {
- SkippedAssumedCallees.push_back(NewCallee);
- SpecializedForAllCallees = false;
- continue;
- }
- SpecializedForAnyCallees = true;
-
- LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
- Instruction *ThenTI =
- SplitBlockAndInsertIfThen(LastCmp, IP, /* Unreachable */ false);
- BasicBlock *CBBB = CB->getParent();
- A.registerManifestAddedBasicBlock(*ThenTI->getParent());
- A.registerManifestAddedBasicBlock(*IP->getParent());
- auto *SplitTI = cast<BranchInst>(LastCmp->getNextNode());
- BasicBlock *ElseBB;
- if (&*IP == CB) {
- ElseBB = BasicBlock::Create(ThenTI->getContext(), "",
- ThenTI->getFunction(), CBBB);
- A.registerManifestAddedBasicBlock(*ElseBB);
- IP = BranchInst::Create(CBBB, ElseBB)->getIterator();
- SplitTI->replaceUsesOfWith(CBBB, ElseBB);
- } else {
- ElseBB = IP->getParent();
- ThenTI->replaceUsesOfWith(ElseBB, CBBB);
- }
- CastInst *RetBC = nullptr;
- CallInst *NewCall = nullptr;
- if (isLegalToPromote(*CB, NewCallee)) {
- auto *CBClone = cast<CallBase>(CB->clone());
- CBClone->insertBefore(ThenTI);
- NewCall = &cast<CallInst>(promoteCall(*CBClone, NewCallee, &RetBC));
- } else {
- NewCall = CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,
- CB->getName(), ThenTI->getIterator());
- }
- NewCalls.push_back({NewCall, RetBC});
- }
-
- auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
- if (!AllCalleesKnown)
- return ChangeStatus::UNCHANGED;
- MDBuilder MDB(IndirectCB.getContext());
- MDNode *Callees = MDB.createCallees(SkippedAssumedCallees);
- IndirectCB.setMetadata(LLVMContext::MD_callees, Callees);
- return ChangeStatus::CHANGED;
- };
-
- if (!SpecializedForAnyCallees)
- return AttachCalleeMetadata(*CB);
-
- // Check if we need the fallback indirect call still.
- if (SpecializedForAllCallees) {
- LastCmp->replaceAllUsesWith(ConstantInt::getTrue(LastCmp->getContext()));
- LastCmp->eraseFromParent();
- new UnreachableInst(IP->getContext(), IP);
- IP->eraseFromParent();
- } else {
- auto *CBClone = cast<CallInst>(CB->clone());
- CBClone->setName(CB->getName());
- CBClone->insertBefore(*IP->getParent(), IP);
- NewCalls.push_back({CBClone, nullptr});
- AttachCalleeMetadata(*CBClone);
- }
-
- // Check if we need a PHI to merge the results.
- if (!CBIsVoid) {
- auto *PHI = PHINode::Create(CB->getType(), NewCalls.size(),
- CB->getName() + ".phi",
- CB->getParent()->getFirstInsertionPt());
- for (auto &It : NewCalls) {
- CallBase *NewCall = It.first;
- Instruction *CallRet = It.second ? It.second : It.first;
- if (CallRet->getType() == CB->getType())
- PHI->addIncoming(CallRet, CallRet->getParent());
- else if (NewCall->getType()->isVoidTy())
- PHI->addIncoming(PoisonValue::get(CB->getType()),
- NewCall->getParent());
- else
- llvm_unreachable("Call return should match or be void!");
- }
- A.changeAfterManifest(IRPosition::callsite_returned(*CB), *PHI);
- }
-
- A.deleteAfterManifest(*CB);
- Changed = ChangeStatus::CHANGED;
-
- return Changed;
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
- " indirect call site with " + std::to_string(AssumedCallees.size()) +
- " functions";
- }
-
- void trackStatistics() const override {
- if (AllCalleesKnown) {
- STATS_DECLTRACK(
- Eliminated, CallSites,
- "Number of indirect call sites eliminated via specialization")
- } else {
- STATS_DECLTRACK(Specialized, CallSites,
- "Number of indirect call sites specialized")
- }
- }
-
- bool foreachCallee(function_ref<bool(Function *)> CB) const override {
- return isValidState() && AllCalleesKnown && all_of(AssumedCallees, CB);
- }
-
-private:
- /// Map to remember filter results.
- DenseMap<Function *, std::optional<bool>> FilterResults;
-
- /// If the !callee metadata was present, this set will contain all potential
- /// callees (superset).
- SmallSetVector<Function *, 4> PotentialCallees;
-
- /// This set contains all currently assumed calllees, which might grow over
- /// time.
- SmallSetVector<Function *, 4> AssumedCallees;
-
- /// Flag to indicate if all possible callees are in the AssumedCallees set or
- /// if there could be others.
- bool AllCalleesKnown = true;
-};
-} // namespace
-
-/// ------------------------ Address Space ------------------------------------
-namespace {
-struct AAAddressSpaceImpl : public AAAddressSpace {
- AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
- : AAAddressSpace(IRP, A) {}
-
- int32_t getAddressSpace() const override {
- assert(isValidState() && "the AA is invalid");
- return AssumedAddressSpace;
- }
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
- "Associated value is not a pointer");
- }
-
- ChangeStatus updateImpl(Attributor &A) override {
- int32_t OldAddressSpace = AssumedAddressSpace;
- auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(getIRPosition(), this,
- DepClassTy::REQUIRED);
- auto Pred = [&](Value &Obj) {
- if (isa<UndefValue>(&Obj))
- return true;
- return takeAddressSpace(Obj.getType()->getPointerAddressSpace());
- };
-
- if (!AUO->forallUnderlyingObjects(Pred))
- return indicatePessimisticFixpoint();
-
- return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
- : ChangeStatus::CHANGED;
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
- Value *AssociatedValue = &getAssociatedValue();
- Value *OriginalValue = peelAddrspacecast(AssociatedValue);
- if (getAddressSpace() == NoAddressSpace ||
- static_cast<uint32_t>(getAddressSpace()) ==
- getAssociatedType()->getPointerAddressSpace())
- return ChangeStatus::UNCHANGED;
-
- Type *NewPtrTy = PointerType::get(getAssociatedType()->getContext(),
- static_cast<uint32_t>(getAddressSpace()));
- bool UseOriginalValue =
- OriginalValue->getType()->getPointerAddressSpace() ==
- static_cast<uint32_t>(getAddressSpace());
-
- bool Changed = false;
-
- auto MakeChange = [&](Instruction *I, Use &U) {
- Changed = true;
- if (UseOriginalValue) {
- A.changeUseAfterManifest(U, *OriginalValue);
- return;
- }
- Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
- CastInst->insertBefore(cast<Instruction>(I));
- A.changeUseAfterManifest(U, *CastInst);
- };
-
- auto Pred = [&](const Use &U, bool &) {
- if (U.get() != AssociatedValue)
- return true;
- auto *Inst = dyn_cast<Instruction>(U.getUser());
- if (!Inst)
- return true;
- // This is a WA to make sure we only change uses from the corresponding
- // CGSCC if the AA is run on CGSCC instead of the entire module.
- if (!A.isRunOn(Inst->getFunction()))
- return true;
- if (isa<LoadInst>(Inst))
- MakeChange(Inst, const_cast<Use &>(U));
- if (isa<StoreInst>(Inst)) {
- // We only make changes if the use is the pointer operand.
- if (U.getOperandNo() == 1)
- MakeChange(Inst, const_cast<Use &>(U));
- }
- return true;
- };
-
- // It doesn't matter if we can't check all uses as we can simply
- // conservatively ignore those that can not be visited.
- (void)A.checkForAllUses(Pred, *this, getAssociatedValue(),
- /* CheckBBLivenessOnly */ true);
-
- return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- if (!isValidState())
- return "addrspace(<invalid>)";
- return "addrspace(" +
- (AssumedAddressSpace == NoAddressSpace
- ? "none"
- : std::to_string(AssumedAddressSpace)) +
- ")";
- }
-
-private:
- int32_t AssumedAddressSpace = NoAddressSpace;
-
- bool takeAddressSpace(int32_t AS) {
- if (AssumedAddressSpace == NoAddressSpace) {
- AssumedAddressSpace = AS;
- return true;
- }
- return AssumedAddressSpace == AS;
- }
-
- static Value *peelAddrspacecast(Value *V) {
- if (auto *I = dyn_cast<AddrSpaceCastInst>(V))
- return peelAddrspacecast(I->getPointerOperand());
- if (auto *C = dyn_cast<ConstantExpr>(V))
- if (C->getOpcode() == Instruction::AddrSpaceCast)
- return peelAddrspacecast(C->getOperand(0));
- return V;
- }
-};
-
-struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
- AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
- : AAAddressSpaceImpl(IRP, A) {}
-
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(addrspace);
- }
-};
-
-struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
- AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
- : AAAddressSpaceImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- // TODO: we don't rewrite function argument for now because it will need to
- // rewrite the function signature and all call sites.
- (void)indicatePessimisticFixpoint();
- }
-
- void trackStatistics() const override {
- STATS_DECLTRACK_FNRET_ATTR(addrspace);
- }
-};
-
-struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
- AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AAAddressSpaceImpl(IRP, A) {}
-
- void trackStatistics() const override {
- STATS_DECLTRACK_CSRET_ATTR(addrspace);
- }
-};
-
-struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
- AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
- : AAAddressSpaceImpl(IRP, A) {}
-
- void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
-};
-
-struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
- AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAAddressSpaceImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- // TODO: we don't rewrite call site argument for now because it will need to
- // rewrite the function signature of the callee.
- (void)indicatePessimisticFixpoint();
- }
-
- void trackStatistics() const override {
- STATS_DECLTRACK_CSARG_ATTR(addrspace);
- }
-};
-} // namespace
-
-/// ----------- Allocation Info ----------
-namespace {
-struct AAAllocationInfoImpl : public AAAllocationInfo {
- AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
- : AAAllocationInfo(IRP, A) {}
-
- std::optional<TypeSize> getAllocatedSize() const override {
- assert(isValidState() && "the AA is invalid");
- return AssumedAllocatedSize;
- }
-
- const NewOffsetsTy &getNewOffsets() const override {
- assert(isValidState() && "the AA is invalid");
- return NewComputedOffsets;
- }
-
- std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
- const DataLayout &DL) {
-
- // TODO: implement case for malloc like instructions
- switch (I->getOpcode()) {
- case Instruction::Alloca: {
- AllocaInst *AI = cast<AllocaInst>(I);
- return AI->getAllocationSize(DL);
- }
- default:
- return std::nullopt;
- }
- }
-
- ChangeStatus updateImpl(Attributor &A) override {
-
- const IRPosition &IRP = getIRPosition();
- Instruction *I = IRP.getCtxI();
-
- // TODO: update check for malloc like calls
- if (!isa<AllocaInst>(I))
- return indicatePessimisticFixpoint();
-
- bool IsKnownNoCapture;
- if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
- A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture))
- return indicatePessimisticFixpoint();
-
- const AAPointerInfo *PI =
- A.getOrCreateAAFor<AAPointerInfo>(IRP, *this, DepClassTy::REQUIRED);
-
- if (!PI)
- return indicatePessimisticFixpoint();
-
- if (!PI->getState().isValidState())
- return indicatePessimisticFixpoint();
-
- const DataLayout &DL = A.getDataLayout();
- const auto AllocationSize = findInitialAllocationSize(I, DL);
-
- // If allocation size is nullopt, we give up.
- if (!AllocationSize)
- return indicatePessimisticFixpoint();
-
- // For zero sized allocations, we give up.
- // Since we can't reduce further
- if (*AllocationSize == 0)
- return indicatePessimisticFixpoint();
-
- int64_t NumBins = PI->numOffsetBins();
-
- if (NumBins == 0) {
- auto NewAllocationSize = std::optional<TypeSize>(TypeSize(0, false));
- if (!changeAllocationSize(NewAllocationSize))
- return ChangeStatus::UNCHANGED;
- return ChangeStatus::CHANGED;
- }
-
- // For each access bin
- // Compute its new start Offset and store the results in a new map
- // (NewOffsetBins).
- int64_t PrevBinEndOffset = 0;
- bool ChangedOffsets = false;
- for (AAPointerInfo::OffsetBinsTy::const_iterator It = PI->begin();
- It != PI->end(); It++) {
- const AA::RangeTy &OldRange = It->getFirst();
- int64_t NewStartOffset = PrevBinEndOffset;
- int64_t NewEndOffset = NewStartOffset + OldRange.Size;
- PrevBinEndOffset = NewEndOffset;
-
- ChangedOffsets |= setNewOffsets(OldRange, OldRange.Offset, NewStartOffset,
- OldRange.Size);
- }
-
- // Set the new size of the allocation, the new size of the Allocation should
- // be the size of NewEndOffset * 8, in bits.
- auto NewAllocationSize =
- std::optional<TypeSize>(TypeSize(PrevBinEndOffset * 8, false));
-
- if (!changeAllocationSize(NewAllocationSize))
- return ChangeStatus::UNCHANGED;
-
- if (!ChangedOffsets)
- return ChangeStatus::UNCHANGED;
-
- return ChangeStatus::CHANGED;
- }
-
- /// See AbstractAttribute::manifest(...).
- ChangeStatus manifest(Attributor &A) override {
-
- assert(isValidState() &&
- "Manifest should only be called if the state is valid.");
-
- bool Changed = false;
- const IRPosition &IRP = getIRPosition();
- Instruction *I = IRP.getCtxI();
-
- auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
-
- unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
- switch (I->getOpcode()) {
- // TODO: add case for malloc like calls
- case Instruction::Alloca: {
-
- AllocaInst *AI = cast<AllocaInst>(I);
-
- Type *CharType = Type::getInt8Ty(I->getContext());
-
- Type *CharArrayType = ArrayType::get(CharType, NumBytesToAllocate);
-
- BasicBlock::iterator insertPt = AI->getIterator();
- insertPt = std::next(insertPt);
- AllocaInst *NewAllocaInst = new AllocaInst(
- CharArrayType, AI->getAddressSpace(), AI->getName(), insertPt);
-
- Changed |= A.changeAfterManifest(IRPosition::inst(*AI), *NewAllocaInst);
- break;
- }
- default:
- break;
- }
-
- const AAPointerInfo *PI =
- A.getOrCreateAAFor<AAPointerInfo>(IRP, *this, DepClassTy::REQUIRED);
-
- if (!PI)
- return ChangeStatus::UNCHANGED;
-
- if (!PI->getState().isValidState())
- return ChangeStatus::UNCHANGED;
-
- const auto &NewOffsetsMap = getNewOffsets();
- for (AAPointerInfo::OffsetBinsTy::const_iterator It = PI->begin();
- It != PI->end(); It++) {
-
- const auto &OldOffsetRange = It->getFirst();
-
- // If the OldOffsetRange is not in the map, offsets for that bin did not
- // change We should just continue and skip changing the offsets in that
- // case
- if (!NewOffsetsMap.contains(OldOffsetRange))
- continue;
-
- const auto &NewOffsetRange = NewOffsetsMap.lookup(OldOffsetRange);
- for (const auto AccIndex : It->getSecond()) {
-
- const auto &AccessInstruction = PI->getBinAccess(AccIndex);
- auto *LocalInst = AccessInstruction.getLocalInst();
-
- switch (LocalInst->getOpcode()) {
- case Instruction::Load: {
- LoadInst *OldLoadInst = cast<LoadInst>(LocalInst);
- Value *PointerOperand = OldLoadInst->getPointerOperand();
-
- IntegerType *Int8TyInteger =
- IntegerType::get(LocalInst->getContext(), 8);
- IntegerType *Int64TyInteger =
- IntegerType::get(LocalInst->getContext(), 64);
- Value *indexList[2] = {
- ConstantInt::get(Int64TyInteger, 0),
- ConstantInt::get(Int64TyInteger,
- NewOffsetRange.Offset - OldOffsetRange.Offset)};
- Value *GepToNewAddress = GetElementPtrInst::Create(
- Int8TyInteger, PointerOperand, indexList, "NewGep", OldLoadInst);
-
- LoadInst *NewLoadInst =
- new LoadInst(OldLoadInst->getType(), GepToNewAddress,
- OldLoadInst->getName(), OldLoadInst);
- Changed |= A.changeAfterManifest(IRPosition::inst(*OldLoadInst),
- *NewLoadInst);
- break;
- }
- case Instruction::Store: {
- StoreInst *OldStoreInst = cast<StoreInst>(LocalInst);
- Value *PointerOperand = OldStoreInst->getPointerOperand();
-
- IntegerType *Int8TyInteger =
- IntegerType::get(LocalInst->getContext(), 8);
- IntegerType *Int64TyInteger =
- IntegerType::get(LocalInst->getContext(), 64);
- Value *indexList[2] = {
- ConstantInt::get(Int64TyInteger, 0),
- ConstantInt::get(Int64TyInteger,
- NewOffsetRange.Offset - OldOffsetRange.Offset)};
- Value *GepToNewAddress = GetElementPtrInst::Create(
- Int8TyInteger, PointerOperand, indexList, "NewGep", OldStoreInst);
-
- StoreInst *NewStoreInst = new StoreInst(
- OldStoreInst->getValueOperand(), GepToNewAddress, OldStoreInst);
- Changed |= A.changeAfterManifest(IRPosition::inst(*OldStoreInst),
- *NewStoreInst);
- break;
- }
- }
- }
- }
-
- if (!Changed)
- return ChangeStatus::UNCHANGED;
- return ChangeStatus::CHANGED;
- }
-
- /// See AbstractAttribute::getAsStr().
- const std::string getAsStr(Attributor *A) const override {
- if (!isValidState())
- return "allocationinfo(<invalid>)";
- return "allocationinfo(" +
- (AssumedAllocatedSize == HasNoAllocationSize
- ? "none"
- : std::to_string(AssumedAllocatedSize->getFixedValue())) +
- ")";
- }
-
- void dumpNewOffsetBins(raw_ostream &O) {
-
- O << "Printing Map from [OldOffsetsRange] : [NewOffsetsRange] if the "
- "offsets changed."
- << "\n";
- const auto &NewOffsetsMap = getNewOffsets();
- for (auto It = NewOffsetsMap.begin(); It != NewOffsetsMap.end(); It++) {
-
- const auto &OldRange = It->getFirst();
- const auto &NewRange = It->getSecond();
-
- O << "[" << OldRange.Offset << "," << OldRange.Offset + OldRange.Size
- << "] : ";
- O << "[" << NewRange.Offset << "," << NewRange.Offset + NewRange.Size
- << "]";
- O << "\n";
- }
- }
-
-private:
- std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
- NewOffsetsTy NewComputedOffsets;
-
- // Maintain the computed allocation size of the object.
- // Returns (bool) weather the size of the allocation was modified or not.
- bool changeAllocationSize(std::optional<TypeSize> Size) {
- if (AssumedAllocatedSize == HasNoAllocationSize ||
- AssumedAllocatedSize != Size) {
- AssumedAllocatedSize = Size;
- return true;
- }
- return false;
- }
-
- // Maps an old byte range to its new Offset range in the new allocation.
- // Returns (bool) weather the old byte range's offsets changed or not.
- bool setNewOffsets(const AA::RangeTy &OldRange, int64_t OldOffset,
- int64_t NewComputedOffset, int64_t Size) {
-
- if (OldOffset == NewComputedOffset)
- return false;
-
- AA::RangeTy &NewRange = NewComputedOffsets.getOrInsertDefault(OldRange);
- NewRange.Offset = NewComputedOffset;
- NewRange.Size = Size;
-
- return true;
- }
-};
-
-struct AAAllocationInfoFloating : AAAllocationInfoImpl {
- AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
- : AAAllocationInfoImpl(IRP, A) {}
-
- void trackStatistics() const override {
- STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
- }
-};
-
-struct AAAllocationInfoReturned : AAAllocationInfoImpl {
- AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
- : AAAllocationInfoImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
- // TODO: we don't rewrite function argument for now because it will need to
- // rewrite the function signature and all call sites
- (void)indicatePessimisticFixpoint();
- }
-
- void trackStatistics() const override {
- STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
- }
-};
-
-struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
- AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
- : AAAllocationInfoImpl(IRP, A) {}
-
- void trackStatistics() const override {
- STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
- }
-};
-
-struct AAAllocationInfoArgument : AAAllocationInfoImpl {
- AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
- : AAAllocationInfoImpl(IRP, A) {}
-
- void trackStatistics() const override {
- STATS_DECLTRACK_ARG_ATTR(allocationinfo);
- }
-};
-
-struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
- AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
- : AAAllocationInfoImpl(IRP, A) {}
-
- /// See AbstractAttribute::initialize(...).
- void initialize(Attributor &A) override {
-
- (void)indicatePessimisticFixpoint();
- }
-
- void trackStatistics() const override {
- STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
- }
-};
-} // namespace
-
-const char AANoUnwind::ID = 0;
-const char AANoSync::ID = 0;
-const char AANoFree::ID = 0;
-const char AANonNull::ID = 0;
-const char AAMustProgress::ID = 0;
-const char AANoRecurse::ID = 0;
-const char AANonConvergent::ID = 0;
-const char AAWillReturn::ID = 0;
-const char AAUndefinedBehavior::ID = 0;
-const char AANoAlias::ID = 0;
-const char AAIntraFnReachability::ID = 0;
-const char AANoReturn::ID = 0;
-const char AAIsDead::ID = 0;
-const char AADereferenceable::ID = 0;
-const char AAAlign::ID = 0;
-const char AAInstanceInfo::ID = 0;
-const char AANoCapture::ID = 0;
-const char AAValueSimplify::ID = 0;
-const char AAHeapToStack::ID = 0;
-const char AAPrivatizablePtr::ID = 0;
-const char AAMemoryBehavior::ID = 0;
-const char AAMemoryLocation::ID = 0;
-const char AAValueConstantRange::ID = 0;
-const char AAPotentialConstantValues::ID = 0;
-const char AAPotentialValues::ID = 0;
-const char AANoUndef::ID = 0;
-const char AANoFPClass::ID = 0;
-const char AACallEdges::ID = 0;
-const char AAInterFnReachability::ID = 0;
-const char AAPointerInfo::ID = 0;
-const char AAAssumptionInfo::ID = 0;
-const char AAUnderlyingObjects::ID = 0;
-const char AAAddressSpace::ID = 0;
-const char AAAllocationInfo::ID = 0;
-const char AAIndirectCallInfo::ID = 0;
-const char AAGlobalValueInfo::ID = 0;
-const char AADenormalFPMath::ID = 0;
-
-// Macro magic to create the static generator function for attributes that
-// follow the naming scheme.
-
-#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
- case IRPosition::PK: \
- llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
-
-#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
- case IRPosition::PK: \
- AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
- ++NumAAs; \
- break;
-
-#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
- CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
- CLASS *AA = nullptr; \
- switch (IRP.getPositionKind()) { \
- SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
- SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
- SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
- SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
- SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
- SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
- } \
- return *AA; \
- }
-
-#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
- CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
- CLASS *AA = nullptr; \
- switch (IRP.getPositionKind()) { \
- SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
- SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
- SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
- } \
- return *AA; \
- }
-
-#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
- CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
- CLASS *AA = nullptr; \
- switch (IRP.getPositionKind()) { \
- SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
- default: \
- llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
- " position!"); \
- } \
- return *AA; \
- }
-
-#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
- CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
- CLASS *AA = nullptr; \
- switch (IRP.getPositionKind()) { \
- SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
- } \
- return *AA; \
- }
-
-#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
- CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
- CLASS *AA = nullptr; \
- switch (IRP.getPositionKind()) { \
- SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
- SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
- SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
- SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
- SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
- SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
- SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
- } \
- return *AA; \
- }
-
-#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
- CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
- CLASS *AA = nullptr; \
- switch (IRP.getPositionKind()) { \
- SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
- SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
- SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
- } \
- return *AA; \
- }
-
-CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)
-CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)
-CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse)
-CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn)
-CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn)
-CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation)
-CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges)
-CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo)
-CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress)
-
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
-
-CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
-CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
-CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)
-CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects)
-
-CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite,
- AAIndirectCallInfo)
-CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating,
- AAGlobalValueInfo)
-
-CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)
-CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior)
-CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent)
-CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability)
-CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability)
-CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath)
-
-CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior)
-
-#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
-#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
-#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
-#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
-#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
-#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
-#undef SWITCH_PK_CREATE
-#undef SWITCH_PK_INV
diff --git a/llvm/test/Transforms/Attributor/allocator.ll b/llvm/test/Transforms/Attributor/allocator.ll
index 860f4a63e6754..dea385f52c204 100644
--- a/llvm/test/Transforms/Attributor/allocator.ll
+++ b/llvm/test/Transforms/Attributor/allocator.ll
@@ -13,14 +13,14 @@ define dso_local void @positive_alloca_1(i32 noundef %val) #0 {
; CHECK-LABEL: define dso_local void @positive_alloca_1
; CHECK-SAME: (i32 noundef [[VAL:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VAL_ADDR:%.*]] = alloca i64, align 4
-; CHECK-NEXT: [[F:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
-; CHECK-NEXT: store i32 [[VAL]], ptr [[VAL_ADDR]], align 4
-; CHECK-NEXT: store i32 10, ptr [[F]], align 4
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[F]], align 4
+; CHECK-NEXT: [[VAL_ADDR1:%.*]] = alloca [4 x i8], align 1
+; CHECK-NEXT: [[F2:%.*]] = alloca [4 x i8], align 1
+; CHECK-NEXT: store i32 [[VAL]], ptr [[VAL_ADDR1]], align 4
+; CHECK-NEXT: store i32 10, ptr [[F2]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[F2]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], 1
-; CHECK-NEXT: store i32 [[ADD]], ptr [[F]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[F]], align 4
+; CHECK-NEXT: store i32 [[ADD]], ptr [[F2]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[F2]], align 4
; CHECK-NEXT: [[ADD3:%.*]] = add nsw i32 [[TMP1]], [[VAL]]
; CHECK-NEXT: [[CALL:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 noundef [[ADD3]])
; CHECK-NEXT: ret void
@@ -49,11 +49,11 @@ define dso_local void @positive_malloc_1(ptr noundef %val) #0 {
; CHECK-LABEL: define dso_local void @positive_malloc_1
; CHECK-SAME: (ptr nocapture nofree noundef readonly [[VAL:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VAL_ADDR:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: [[F:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: store ptr [[VAL]], ptr [[VAL_ADDR]], align 8
+; CHECK-NEXT: [[VAL_ADDR1:%.*]] = alloca [8 x i8], align 1
+; CHECK-NEXT: [[F2:%.*]] = alloca [8 x i8], align 1
+; CHECK-NEXT: store ptr [[VAL]], ptr [[VAL_ADDR1]], align 8
; CHECK-NEXT: [[CALL:%.*]] = call noalias ptr @malloc(i64 noundef 12)
-; CHECK-NEXT: store ptr [[CALL]], ptr [[F]], align 8
+; CHECK-NEXT: store ptr [[CALL]], ptr [[F2]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[VAL]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP0]], 10
; CHECK-NEXT: store i32 [[ADD]], ptr [[CALL]], align 4
@@ -86,11 +86,11 @@ define dso_local void @positive_malloc_2(ptr noundef %val) #0 {
; CHECK-LABEL: define dso_local void @positive_malloc_2
; CHECK-SAME: (ptr nocapture nofree noundef readonly [[VAL:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VAL_ADDR:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: [[F:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: store ptr [[VAL]], ptr [[VAL_ADDR]], align 8
+; CHECK-NEXT: [[VAL_ADDR1:%.*]] = alloca [8 x i8], align 1
+; CHECK-NEXT: [[F3:%.*]] = alloca [8 x i8], align 1
+; CHECK-NEXT: store ptr [[VAL]], ptr [[VAL_ADDR1]], align 8
; CHECK-NEXT: [[CALL:%.*]] = call noalias ptr @malloc(i64 noundef 60)
-; CHECK-NEXT: store ptr [[CALL]], ptr [[F]], align 8
+; CHECK-NEXT: store ptr [[CALL]], ptr [[F3]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[VAL]], align 4
; CHECK-NEXT: store i32 [[TMP0]], ptr [[CALL]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[CALL]], align 4
@@ -125,19 +125,19 @@ define dso_local ptr @negative_test_escaping_pointer(i32 noundef %val) #0 {
; CHECK-LABEL: define dso_local ptr @negative_test_escaping_pointer
; CHECK-SAME: (i32 noundef [[VAL:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VAL_ADDR:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[F:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: store i32 [[VAL]], ptr [[VAL_ADDR]], align 4
+; CHECK-NEXT: [[VAL_ADDR1:%.*]] = alloca [4 x i8], align 1
+; CHECK-NEXT: [[F2:%.*]] = alloca [8 x i8], align 1
+; CHECK-NEXT: store i32 [[VAL]], ptr [[VAL_ADDR1]], align 4
; CHECK-NEXT: [[CALL:%.*]] = call noalias ptr @malloc(i64 noundef 16)
-; CHECK-NEXT: store ptr [[CALL]], ptr [[F]], align 8
-; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[F]], align 8
+; CHECK-NEXT: store ptr [[CALL]], ptr [[F2]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[F2]], align 8
; CHECK-NEXT: store i32 2, ptr [[TMP0]], align 8
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 10, [[VAL]]
-; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[F]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[F2]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 8
; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP2]], [[ADD]]
; CHECK-NEXT: store i32 [[ADD2]], ptr [[TMP1]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[F]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[F2]], align 8
; CHECK-NEXT: ret ptr [[TMP3]]
;
entry:
@@ -168,14 +168,27 @@ define dso_local void @positive_test_not_a_single_start_offset(i32 noundef %val)
; CHECK-LABEL: define dso_local void @positive_test_not_a_single_start_offset
; CHECK-SAME: (i32 noundef [[VAL:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VAL_ADDR:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[F:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
-; CHECK-NEXT: store i32 [[VAL]], ptr [[VAL_ADDR]], align 4
+; CHECK-NEXT: [[VAL_ADDR1:%.*]] = alloca [4 x i8], align 1
+; CHECK-NEXT: [[F2:%.*]] = alloca [5 x i8], align 1
+; CHECK-NEXT: store i32 [[VAL]], ptr [[VAL_ADDR1]], align 4
; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 2, [[VAL]]
-; CHECK-NEXT: store i32 [[MUL]], ptr [[F]], align 4
-; CHECK-NEXT: [[CALL:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 noundef [[MUL]])
-; CHECK-NEXT: [[C:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr [[F]], i32 0, i32 2
-; CHECK-NEXT: [[CALL3:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 noundef 67)
+; CHECK-NEXT: store i32 [[MUL]], ptr [[F2]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[F2]], align 4
+; CHECK-NEXT: [[CALL:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 noundef [[TMP0]])
+; CHECK-NEXT: [[C:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr [[F2]], i32 0, i32 2
+; CHECK-NEXT: [[CONV1:%.*]] = trunc i32 [[TMP0]] to i8
+; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[C]] to i32
+; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[TMP1]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[TMP2]] to ptr
+; CHECK-NEXT: store i8 [[CONV1]], ptr [[TMP3]], align 1
+; CHECK-NEXT: store i8 [[CONV1]], ptr [[C]], align 4
+; CHECK-NEXT: [[C2:%.*]] = getelementptr inbounds [[STRUCT_FOO]], ptr [[F2]], i32 0, i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = ptrtoint ptr [[C2]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = sub i32 [[TMP4]], 4
+; CHECK-NEXT: [[TMP6:%.*]] = inttoptr i32 [[TMP5]] to ptr
+; CHECK-NEXT: [[TMP7:%.*]] = load i8, ptr [[TMP6]], align 1
+; CHECK-NEXT: [[CONV:%.*]] = sext i8 [[TMP7]] to i32
+; CHECK-NEXT: [[CALL3:%.*]] = call i32 (ptr, ...) @printf(ptr noundef nonnull dereferenceable(17) @.str, i32 noundef [[CONV]])
; CHECK-NEXT: ret void
;
entry:
@@ -190,7 +203,8 @@ entry:
%1 = load i32, ptr %a1, align 4
%call = call i32 (ptr, ...) @printf(ptr noundef @.str, i32 noundef %1)
%c = getelementptr inbounds %struct.Foo, ptr %f, i32 0, i32 2
- store i8 67, ptr %c, align 4
+ %conv1 = trunc i32 %1 to i8
+ store i8 %conv1, ptr %c, align 4
%c2 = getelementptr inbounds %struct.Foo, ptr %f, i32 0, i32 2
%2 = load i8, ptr %c2, align 4
%conv = sext i8 %2 to i32
@@ -202,7 +216,7 @@ entry:
define dso_local void @positive_test_reduce_array_allocation_1() {
; CHECK-LABEL: define dso_local void @positive_test_reduce_array_allocation_1() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[ARRAY1:%.*]] = alloca ptr, i32 10, align 8
+; CHECK-NEXT: [[ARRAY1:%.*]] = alloca [4 x i8], align 1
; CHECK-NEXT: store i32 0, ptr [[ARRAY1]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAY1]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], 2
@@ -239,15 +253,15 @@ define dso_local void @baz(ptr noundef %val, i32 noundef %arrayLength) #0 {
; CHECK-LABEL: define dso_local void @baz
; CHECK-SAME: (ptr nocapture nofree noundef readonly [[VAL:%.*]], i32 noundef [[ARRAYLENGTH:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[VAL_ADDR:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: [[ARRAYLENGTH_ADDR:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[F:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: store ptr [[VAL]], ptr [[VAL_ADDR]], align 8
-; CHECK-NEXT: store i32 [[ARRAYLENGTH]], ptr [[ARRAYLENGTH_ADDR]], align 4
+; CHECK-NEXT: [[VAL_ADDR1:%.*]] = alloca [8 x i8], align 1
+; CHECK-NEXT: [[ARRAYLENGTH_ADDR2:%.*]] = alloca [4 x i8], align 1
+; CHECK-NEXT: [[F3:%.*]] = alloca [8 x i8], align 1
+; CHECK-NEXT: store ptr [[VAL]], ptr [[VAL_ADDR1]], align 8
+; CHECK-NEXT: store i32 [[ARRAYLENGTH]], ptr [[ARRAYLENGTH_ADDR2]], align 4
; CHECK-NEXT: [[CONV:%.*]] = sext i32 [[ARRAYLENGTH]] to i64
; CHECK-NEXT: [[MUL:%.*]] = mul i64 4, [[CONV]]
; CHECK-NEXT: [[CALL:%.*]] = call noalias ptr @malloc(i64 noundef [[MUL]])
-; CHECK-NEXT: store ptr [[CALL]], ptr [[F]], align 8
+; CHECK-NEXT: store ptr [[CALL]], ptr [[F3]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[VAL]], align 4
; CHECK-NEXT: store i32 [[TMP0]], ptr [[CALL]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[CALL]], align 4
@@ -283,8 +297,8 @@ entry:
define dso_local void @positive_test_reduce_array_allocation_2() #0 {
; CHECK-LABEL: define dso_local void @positive_test_reduce_array_allocation_2() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[ARRAY1:%.*]] = alloca ptr, align 8
-; CHECK-NEXT: [[I2:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[ARRAY1:%.*]] = alloca [8 x i8], align 1
+; CHECK-NEXT: [[I2:%.*]] = alloca [4 x i8], align 1
; CHECK-NEXT: [[CALL:%.*]] = call noalias ptr @malloc(i64 noundef 40000)
; CHECK-NEXT: store ptr [[CALL]], ptr [[ARRAY1]], align 8
; CHECK-NEXT: store i32 0, ptr [[I2]], align 4
@@ -433,20 +447,21 @@ define dso_local void @pthread_test(){
; TUNIT-LABEL: define dso_local void @pthread_test() {
; TUNIT-NEXT: [[ARG1:%.*]] = alloca i8, align 8
; TUNIT-NEXT: [[THREAD:%.*]] = alloca i64, align 8
-; TUNIT-NEXT: [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_remain_same, ptr noundef nonnull align 8 dereferenceable(1) [[ARG1]])
-; TUNIT-NEXT: [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(12) undef)
+; TUNIT-NEXT: [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_remain_same, ptr noundef nonnull align 8 dereferenceable(1) [[ARG1]])
+; TUNIT-NEXT: [[F1:%.*]] = alloca [4 x i8], align 1
+; TUNIT-NEXT: [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr noalias nocapture nofree nonnull readnone align 4 dereferenceable(12) undef)
; TUNIT-NEXT: [[F2:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
-; TUNIT-NEXT: [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_check_captured_pointer, ptr noundef nonnull align 4 dereferenceable(12) [[F2]])
+; TUNIT-NEXT: [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_check_captured_pointer, ptr noundef nonnull align 4 dereferenceable(12) [[F2]])
; TUNIT-NEXT: ret void
;
; CGSCC-LABEL: define dso_local void @pthread_test() {
; CGSCC-NEXT: [[ARG1:%.*]] = alloca i8, align 8
; CGSCC-NEXT: [[THREAD:%.*]] = alloca i64, align 8
-; CGSCC-NEXT: [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_remain_same, ptr noundef nonnull align 8 dereferenceable(1) [[ARG1]])
+; CGSCC-NEXT: [[CALL1:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_remain_same, ptr noundef nonnull align 8 dereferenceable(1) [[ARG1]])
; CGSCC-NEXT: [[F:%.*]] = alloca [[STRUCT_FOO:%.*]], align 4
-; CGSCC-NEXT: [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr nofree noundef nonnull readonly align 4 dereferenceable(12) [[F]])
+; CGSCC-NEXT: [[CALL2:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_allocation_should_be_reduced, ptr noalias nocapture nofree noundef nonnull readonly align 4 dereferenceable(12) [[F]])
; CGSCC-NEXT: [[F2:%.*]] = alloca [[STRUCT_FOO]], align 4
-; CGSCC-NEXT: [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noalias nocapture noundef align 4294967296 null, ptr noundef nonnull @pthread_check_captured_pointer, ptr noundef nonnull align 4 dereferenceable(12) [[F2]])
+; CGSCC-NEXT: [[CALL3:%.*]] = call i32 @pthread_create(ptr noundef nonnull align 8 dereferenceable(8) [[THREAD]], ptr noundef align 4294967296 null, ptr noundef nonnull @pthread_check_captured_pointer, ptr noundef nonnull align 4 dereferenceable(12) [[F2]])
; CGSCC-NEXT: ret void
;
%arg1 = alloca i8, align 8
@@ -511,19 +526,19 @@ define dso_local void @alloca_array_multi_offset(){
; CHECK-LABEL: define dso_local void @alloca_array_multi_offset
; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
-; CHECK-NEXT: store i32 0, ptr [[I]], align 4
+; CHECK-NEXT: [[I2:%.*]] = alloca [4 x i8], align 1
+; CHECK-NEXT: store i32 0, ptr [[I2]], align 4
; CHECK-NEXT: br label [[FOR_COND:%.*]]
; CHECK: for.cond:
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
; CHECK-NEXT: br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
; CHECK: for.body:
; CHECK-NEXT: br label [[FOR_INC:%.*]]
; CHECK: for.inc:
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I2]], align 4
; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP1]], 2
-; CHECK-NEXT: store i32 [[ADD]], ptr [[I]], align 4
+; CHECK-NEXT: store i32 [[ADD]], ptr [[I2]], align 4
; CHECK-NEXT: br label [[FOR_COND]]
; CHECK: for.end:
; CHECK-NEXT: ret void
@@ -570,9 +585,9 @@ declare i32 @printf(ptr noundef, ...) #1
; Function Attrs: nounwind allocsize(0)
declare noalias ptr @malloc(i64 noundef) #1
;.
-; TUNIT: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
+; TUNIT: attributes #[[ATTR0]] = { nofree norecurse nosync nounwind memory(none) }
;.
-; CGSCC: attributes #[[ATTR0]] = { mustprogress nofree norecurse nosync nounwind willreturn memory(none) }
+; CGSCC: attributes #[[ATTR0]] = { nofree norecurse nosync nounwind memory(none) }
;.
; TUNIT: [[META0:![0-9]+]] = !{[[META1:![0-9]+]]}
; TUNIT: [[META1]] = !{i64 2, i64 3, i1 false}
>From 83f9d7458d19a3698184bab3cfabab5cad7767ff Mon Sep 17 00:00:00 2001
From: Vidush Singhal <singhal2 at ruby963.llnl.gov>
Date: Thu, 23 May 2024 15:57:11 -0700
Subject: [PATCH 3/3] Fix files that got accidently deleted
---
llvm/include/llvm/Transforms/IPO/Attributor.h | 6483 ++++++++
.../Transforms/IPO/AttributorAttributes.cpp | 13228 ++++++++++++++++
2 files changed, 19711 insertions(+)
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index e69de29bb2d1d..fb96e52121f82 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -0,0 +1,6483 @@
+//===- Attributor.h --- Module-wide attribute deduction ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Attributor: An inter procedural (abstract) "attribute" deduction framework.
+//
+// The Attributor framework is an inter procedural abstract analysis (fixpoint
+// iteration analysis). The goal is to allow easy deduction of new attributes as
+// well as information exchange between abstract attributes in-flight.
+//
+// The Attributor class is the driver and the link between the various abstract
+// attributes. The Attributor will iterate until a fixpoint state is reached by
+// all abstract attributes in-flight, or until it will enforce a pessimistic fix
+// point because an iteration limit is reached.
+//
+// Abstract attributes, derived from the AbstractAttribute class, actually
+// describe properties of the code. They can correspond to actual LLVM-IR
+// attributes, or they can be more general, ultimately unrelated to LLVM-IR
+// attributes. The latter is useful when an abstract attributes provides
+// information to other abstract attributes in-flight but we might not want to
+// manifest the information. The Attributor allows to query in-flight abstract
+// attributes through the `Attributor::getAAFor` method (see the method
+// description for an example). If the method is used by an abstract attribute
+// P, and it results in an abstract attribute Q, the Attributor will
+// automatically capture a potential dependence from Q to P. This dependence
+// will cause P to be reevaluated whenever Q changes in the future.
+//
+// The Attributor will only reevaluate abstract attributes that might have
+// changed since the last iteration. That means that the Attribute will not
+// revisit all instructions/blocks/functions in the module but only query
+// an update from a subset of the abstract attributes.
+//
+// The update method `AbstractAttribute::updateImpl` is implemented by the
+// specific "abstract attribute" subclasses. The method is invoked whenever the
+// currently assumed state (see the AbstractState class) might not be valid
+// anymore. This can, for example, happen if the state was dependent on another
+// abstract attribute that changed. In every invocation, the update method has
+// to adjust the internal state of an abstract attribute to a point that is
+// justifiable by the underlying IR and the current state of abstract attributes
+// in-flight. Since the IR is given and assumed to be valid, the information
+// derived from it can be assumed to hold. However, information derived from
+// other abstract attributes is conditional on various things. If the justifying
+// state changed, the `updateImpl` has to revisit the situation and potentially
+// find another justification or limit the optimistic assumes made.
+//
+// Change is the key in this framework. Until a state of no-change, thus a
+// fixpoint, is reached, the Attributor will query the abstract attributes
+// in-flight to re-evaluate their state. If the (current) state is too
+// optimistic, hence it cannot be justified anymore through other abstract
+// attributes or the state of the IR, the state of the abstract attribute will
+// have to change. Generally, we assume abstract attribute state to be a finite
+// height lattice and the update function to be monotone. However, these
+// conditions are not enforced because the iteration limit will guarantee
+// termination. If an optimistic fixpoint is reached, or a pessimistic fix
+// point is enforced after a timeout, the abstract attributes are tasked to
+// manifest their result in the IR for passes to come.
+//
+// Attribute manifestation is not mandatory. If desired, there is support to
+// generate a single or multiple LLVM-IR attributes already in the helper struct
+// IRAttribute. In the simplest case, a subclass inherits from IRAttribute with
+// a proper Attribute::AttrKind as template parameter. The Attributor
+// manifestation framework will then create and place a new attribute if it is
+// allowed to do so (based on the abstract state). Other use cases can be
+// achieved by overloading AbstractAttribute or IRAttribute methods.
+//
+//
+// The "mechanics" of adding a new "abstract attribute":
+// - Define a class (transitively) inheriting from AbstractAttribute and one
+// (which could be the same) that (transitively) inherits from AbstractState.
+// For the latter, consider the already available BooleanState and
+// {Inc,Dec,Bit}IntegerState if they fit your needs, e.g., you require only a
+// number tracking or bit-encoding.
+// - Implement all pure methods. Also use overloading if the attribute is not
+// conforming with the "default" behavior: A (set of) LLVM-IR attribute(s) for
+// an argument, call site argument, function return value, or function. See
+// the class and method descriptions for more information on the two
+// "Abstract" classes and their respective methods.
+// - Register opportunities for the new abstract attribute in the
+// `Attributor::identifyDefaultAbstractAttributes` method if it should be
+// counted as a 'default' attribute.
+// - Add sufficient tests.
+// - Add a Statistics object for bookkeeping. If it is a simple (set of)
+// attribute(s) manifested through the Attributor manifestation framework, see
+// the bookkeeping function in Attributor.cpp.
+// - If instructions with a certain opcode are interesting to the attribute, add
+// that opcode to the switch in `Attributor::identifyAbstractAttributes`. This
+// will make it possible to query all those instructions through the
+// `InformationCache::getOpcodeInstMapForFunction` interface and eliminate the
+// need to traverse the IR repeatedly.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
+#define LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
+
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/GraphTraits.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/iterator.h"
+#include "llvm/Analysis/AssumeBundleQueries.h"
+#include "llvm/Analysis/CFG.h"
+#include "llvm/Analysis/CGSCCPassManager.h"
+#include "llvm/Analysis/LazyCallGraph.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MustExecute.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/PostDominators.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/IR/AbstractCallSite.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/ConstantRange.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/DOTGraphTraits.h"
+#include "llvm/Support/DebugCounter.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/ModRef.h"
+#include "llvm/Support/TimeProfiler.h"
+#include "llvm/Support/TypeSize.h"
+#include "llvm/TargetParser/Triple.h"
+#include "llvm/Transforms/Utils/CallGraphUpdater.h"
+
+#include <limits>
+#include <map>
+#include <optional>
+
+namespace llvm {
+
+class DataLayout;
+class LLVMContext;
+class Pass;
+template <typename Fn> class function_ref;
+struct AADepGraphNode;
+struct AADepGraph;
+struct Attributor;
+struct AbstractAttribute;
+struct InformationCache;
+struct AAIsDead;
+struct AttributorCallGraph;
+struct IRPosition;
+
+class Function;
+
+/// Abstract Attribute helper functions.
+namespace AA {
+using InstExclusionSetTy = SmallPtrSet<Instruction *, 4>;
+
+enum class GPUAddressSpace : unsigned {
+ Generic = 0,
+ Global = 1,
+ Shared = 3,
+ Constant = 4,
+ Local = 5,
+};
+
+/// Return true iff \p M target a GPU (and we can use GPU AS reasoning).
+bool isGPU(const Module &M);
+
+/// Flags to distinguish intra-procedural queries from *potentially*
+/// inter-procedural queries. Not that information can be valid for both and
+/// therefore both bits might be set.
+enum ValueScope : uint8_t {
+ Intraprocedural = 1,
+ Interprocedural = 2,
+ AnyScope = Intraprocedural | Interprocedural,
+};
+
+struct ValueAndContext : public std::pair<Value *, const Instruction *> {
+ using Base = std::pair<Value *, const Instruction *>;
+ ValueAndContext(const Base &B) : Base(B) {}
+ ValueAndContext(Value &V, const Instruction *CtxI) : Base(&V, CtxI) {}
+ ValueAndContext(Value &V, const Instruction &CtxI) : Base(&V, &CtxI) {}
+
+ Value *getValue() const { return this->first; }
+ const Instruction *getCtxI() const { return this->second; }
+};
+
+/// Return true if \p I is a `nosync` instruction. Use generic reasoning and
+/// potentially the corresponding AANoSync.
+bool isNoSyncInst(Attributor &A, const Instruction &I,
+ const AbstractAttribute &QueryingAA);
+
+/// Return true if \p V is dynamically unique, that is, there are no two
+/// "instances" of \p V at runtime with different values.
+/// Note: If \p ForAnalysisOnly is set we only check that the Attributor will
+/// never use \p V to represent two "instances" not that \p V could not
+/// technically represent them.
+bool isDynamicallyUnique(Attributor &A, const AbstractAttribute &QueryingAA,
+ const Value &V, bool ForAnalysisOnly = true);
+
+/// Return true if \p V is a valid value in \p Scope, that is a constant or an
+/// instruction/argument of \p Scope.
+bool isValidInScope(const Value &V, const Function *Scope);
+
+/// Return true if the value of \p VAC is a valid at the position of \p VAC,
+/// that is a constant, an argument of the same function, or an instruction in
+/// that function that dominates the position.
+bool isValidAtPosition(const ValueAndContext &VAC, InformationCache &InfoCache);
+
+/// Try to convert \p V to type \p Ty without introducing new instructions. If
+/// this is not possible return `nullptr`. Note: this function basically knows
+/// how to cast various constants.
+Value *getWithType(Value &V, Type &Ty);
+
+/// Return the combination of \p A and \p B such that the result is a possible
+/// value of both. \p B is potentially casted to match the type \p Ty or the
+/// type of \p A if \p Ty is null.
+///
+/// Examples:
+/// X + none => X
+/// not_none + undef => not_none
+/// V1 + V2 => nullptr
+std::optional<Value *>
+combineOptionalValuesInAAValueLatice(const std::optional<Value *> &A,
+ const std::optional<Value *> &B, Type *Ty);
+
+/// Helper to represent an access offset and size, with logic to deal with
+/// uncertainty and check for overlapping accesses.
+struct RangeTy {
+ int64_t Offset = Unassigned;
+ int64_t Size = Unassigned;
+
+ RangeTy(int64_t Offset, int64_t Size) : Offset(Offset), Size(Size) {}
+ RangeTy() = default;
+ static RangeTy getUnknown() { return RangeTy{Unknown, Unknown}; }
+
+ /// Return true if offset or size are unknown.
+ bool offsetOrSizeAreUnknown() const {
+ return Offset == RangeTy::Unknown || Size == RangeTy::Unknown;
+ }
+
+ /// Return true if offset and size are unknown, thus this is the default
+ /// unknown object.
+ bool offsetAndSizeAreUnknown() const {
+ return Offset == RangeTy::Unknown && Size == RangeTy::Unknown;
+ }
+
+ /// Return true if the offset and size are unassigned.
+ bool isUnassigned() const {
+ assert((Offset == RangeTy::Unassigned) == (Size == RangeTy::Unassigned) &&
+ "Inconsistent state!");
+ return Offset == RangeTy::Unassigned;
+ }
+
+ /// Return true if this offset and size pair might describe an address that
+ /// overlaps with \p Range.
+ bool mayOverlap(const RangeTy &Range) const {
+ // Any unknown value and we are giving up -> overlap.
+ if (offsetOrSizeAreUnknown() || Range.offsetOrSizeAreUnknown())
+ return true;
+
+ // Check if one offset point is in the other interval [offset,
+ // offset+size].
+ return Range.Offset + Range.Size > Offset && Range.Offset < Offset + Size;
+ }
+
+ RangeTy &operator&=(const RangeTy &R) {
+ if (R.isUnassigned())
+ return *this;
+ if (isUnassigned())
+ return *this = R;
+ if (Offset == Unknown || R.Offset == Unknown)
+ Offset = Unknown;
+ if (Size == Unknown || R.Size == Unknown)
+ Size = Unknown;
+ if (offsetAndSizeAreUnknown())
+ return *this;
+ if (Offset == Unknown) {
+ Size = std::max(Size, R.Size);
+ } else if (Size == Unknown) {
+ Offset = std::min(Offset, R.Offset);
+ } else {
+ Offset = std::min(Offset, R.Offset);
+ Size = std::max(Offset + Size, R.Offset + R.Size) - Offset;
+ }
+ return *this;
+ }
+
+ /// Comparison for sorting ranges by offset.
+ ///
+ /// Returns true if the offset \p L is less than that of \p R.
+ inline static bool OffsetLessThan(const RangeTy &L, const RangeTy &R) {
+ return L.Offset < R.Offset;
+ }
+
+ /// Constants used to represent special offsets or sizes.
+ /// - We cannot assume that Offsets and Size are non-negative.
+ /// - The constants should not clash with DenseMapInfo, such as EmptyKey
+ /// (INT64_MAX) and TombstoneKey (INT64_MIN).
+ /// We use values "in the middle" of the 64 bit range to represent these
+ /// special cases.
+ static constexpr int64_t Unassigned = std::numeric_limits<int32_t>::min();
+ static constexpr int64_t Unknown = std::numeric_limits<int32_t>::max();
+};
+
+inline raw_ostream &operator<<(raw_ostream &OS, const RangeTy &R) {
+ OS << "[" << R.Offset << ", " << R.Size << "]";
+ return OS;
+}
+
+inline bool operator==(const RangeTy &A, const RangeTy &B) {
+ return A.Offset == B.Offset && A.Size == B.Size;
+}
+
+inline bool operator!=(const RangeTy &A, const RangeTy &B) { return !(A == B); }
+
+/// Return the initial value of \p Obj with type \p Ty if that is a constant.
+Constant *getInitialValueForObj(Attributor &A,
+ const AbstractAttribute &QueryingAA, Value &Obj,
+ Type &Ty, const TargetLibraryInfo *TLI,
+ const DataLayout &DL,
+ RangeTy *RangePtr = nullptr);
+
+/// Collect all potential values \p LI could read into \p PotentialValues. That
+/// is, the only values read by \p LI are assumed to be known and all are in
+/// \p PotentialValues. \p PotentialValueOrigins will contain all the
+/// instructions that might have put a potential value into \p PotentialValues.
+/// Dependences onto \p QueryingAA are properly tracked, \p
+/// UsedAssumedInformation will inform the caller if assumed information was
+/// used.
+///
+/// \returns True if the assumed potential copies are all in \p PotentialValues,
+/// false if something went wrong and the copies could not be
+/// determined.
+bool getPotentiallyLoadedValues(
+ Attributor &A, LoadInst &LI, SmallSetVector<Value *, 4> &PotentialValues,
+ SmallSetVector<Instruction *, 4> &PotentialValueOrigins,
+ const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation,
+ bool OnlyExact = false);
+
+/// Collect all potential values of the one stored by \p SI into
+/// \p PotentialCopies. That is, the only copies that were made via the
+/// store are assumed to be known and all are in \p PotentialCopies. Dependences
+/// onto \p QueryingAA are properly tracked, \p UsedAssumedInformation will
+/// inform the caller if assumed information was used.
+///
+/// \returns True if the assumed potential copies are all in \p PotentialCopies,
+/// false if something went wrong and the copies could not be
+/// determined.
+bool getPotentialCopiesOfStoredValue(
+ Attributor &A, StoreInst &SI, SmallSetVector<Value *, 4> &PotentialCopies,
+ const AbstractAttribute &QueryingAA, bool &UsedAssumedInformation,
+ bool OnlyExact = false);
+
+/// Return true if \p IRP is readonly. This will query respective AAs that
+/// deduce the information and introduce dependences for \p QueryingAA.
+bool isAssumedReadOnly(Attributor &A, const IRPosition &IRP,
+ const AbstractAttribute &QueryingAA, bool &IsKnown);
+
+/// Return true if \p IRP is readnone. This will query respective AAs that
+/// deduce the information and introduce dependences for \p QueryingAA.
+bool isAssumedReadNone(Attributor &A, const IRPosition &IRP,
+ const AbstractAttribute &QueryingAA, bool &IsKnown);
+
+/// Return true if \p ToI is potentially reachable from \p FromI without running
+/// into any instruction in \p ExclusionSet The two instructions do not need to
+/// be in the same function. \p GoBackwardsCB can be provided to convey domain
+/// knowledge about the "lifespan" the user is interested in. By default, the
+/// callers of \p FromI are checked as well to determine if \p ToI can be
+/// reached. If the query is not interested in callers beyond a certain point,
+/// e.g., a GPU kernel entry or the function containing an alloca, the
+/// \p GoBackwardsCB should return false.
+bool isPotentiallyReachable(
+ Attributor &A, const Instruction &FromI, const Instruction &ToI,
+ const AbstractAttribute &QueryingAA,
+ const AA::InstExclusionSetTy *ExclusionSet = nullptr,
+ std::function<bool(const Function &F)> GoBackwardsCB = nullptr);
+
+/// Same as above but it is sufficient to reach any instruction in \p ToFn.
+bool isPotentiallyReachable(
+ Attributor &A, const Instruction &FromI, const Function &ToFn,
+ const AbstractAttribute &QueryingAA,
+ const AA::InstExclusionSetTy *ExclusionSet = nullptr,
+ std::function<bool(const Function &F)> GoBackwardsCB = nullptr);
+
+/// Return true if \p Obj is assumed to be a thread local object.
+bool isAssumedThreadLocalObject(Attributor &A, Value &Obj,
+ const AbstractAttribute &QueryingAA);
+
+/// Return true if \p I is potentially affected by a barrier.
+bool isPotentiallyAffectedByBarrier(Attributor &A, const Instruction &I,
+ const AbstractAttribute &QueryingAA);
+bool isPotentiallyAffectedByBarrier(Attributor &A, ArrayRef<const Value *> Ptrs,
+ const AbstractAttribute &QueryingAA,
+ const Instruction *CtxI);
+} // namespace AA
+
+template <>
+struct DenseMapInfo<AA::ValueAndContext>
+ : public DenseMapInfo<AA::ValueAndContext::Base> {
+ using Base = DenseMapInfo<AA::ValueAndContext::Base>;
+ static inline AA::ValueAndContext getEmptyKey() {
+ return Base::getEmptyKey();
+ }
+ static inline AA::ValueAndContext getTombstoneKey() {
+ return Base::getTombstoneKey();
+ }
+ static unsigned getHashValue(const AA::ValueAndContext &VAC) {
+ return Base::getHashValue(VAC);
+ }
+
+ static bool isEqual(const AA::ValueAndContext &LHS,
+ const AA::ValueAndContext &RHS) {
+ return Base::isEqual(LHS, RHS);
+ }
+};
+
+template <>
+struct DenseMapInfo<AA::ValueScope> : public DenseMapInfo<unsigned char> {
+ using Base = DenseMapInfo<unsigned char>;
+ static inline AA::ValueScope getEmptyKey() {
+ return AA::ValueScope(Base::getEmptyKey());
+ }
+ static inline AA::ValueScope getTombstoneKey() {
+ return AA::ValueScope(Base::getTombstoneKey());
+ }
+ static unsigned getHashValue(const AA::ValueScope &S) {
+ return Base::getHashValue(S);
+ }
+
+ static bool isEqual(const AA::ValueScope &LHS, const AA::ValueScope &RHS) {
+ return Base::isEqual(LHS, RHS);
+ }
+};
+
+template <>
+struct DenseMapInfo<const AA::InstExclusionSetTy *>
+ : public DenseMapInfo<void *> {
+ using super = DenseMapInfo<void *>;
+ static inline const AA::InstExclusionSetTy *getEmptyKey() {
+ return static_cast<const AA::InstExclusionSetTy *>(super::getEmptyKey());
+ }
+ static inline const AA::InstExclusionSetTy *getTombstoneKey() {
+ return static_cast<const AA::InstExclusionSetTy *>(
+ super::getTombstoneKey());
+ }
+ static unsigned getHashValue(const AA::InstExclusionSetTy *BES) {
+ unsigned H = 0;
+ if (BES)
+ for (const auto *II : *BES)
+ H += DenseMapInfo<const Instruction *>::getHashValue(II);
+ return H;
+ }
+ static bool isEqual(const AA::InstExclusionSetTy *LHS,
+ const AA::InstExclusionSetTy *RHS) {
+ if (LHS == RHS)
+ return true;
+ if (LHS == getEmptyKey() || RHS == getEmptyKey() ||
+ LHS == getTombstoneKey() || RHS == getTombstoneKey())
+ return false;
+ auto SizeLHS = LHS ? LHS->size() : 0;
+ auto SizeRHS = RHS ? RHS->size() : 0;
+ if (SizeLHS != SizeRHS)
+ return false;
+ if (SizeRHS == 0)
+ return true;
+ return llvm::set_is_subset(*LHS, *RHS);
+ }
+};
+
+/// The value passed to the line option that defines the maximal initialization
+/// chain length.
+extern unsigned MaxInitializationChainLength;
+
+///{
+enum class ChangeStatus {
+ CHANGED,
+ UNCHANGED,
+};
+
+ChangeStatus operator|(ChangeStatus l, ChangeStatus r);
+ChangeStatus &operator|=(ChangeStatus &l, ChangeStatus r);
+ChangeStatus operator&(ChangeStatus l, ChangeStatus r);
+ChangeStatus &operator&=(ChangeStatus &l, ChangeStatus r);
+
+enum class DepClassTy {
+ REQUIRED, ///< The target cannot be valid if the source is not.
+ OPTIONAL, ///< The target may be valid if the source is not.
+ NONE, ///< Do not track a dependence between source and target.
+};
+///}
+
+/// The data structure for the nodes of a dependency graph
+struct AADepGraphNode {
+public:
+ virtual ~AADepGraphNode() = default;
+ using DepTy = PointerIntPair<AADepGraphNode *, 1>;
+ using DepSetTy = SmallSetVector<DepTy, 2>;
+
+protected:
+ /// Set of dependency graph nodes which should be updated if this one
+ /// is updated. The bit encodes if it is optional.
+ DepSetTy Deps;
+
+ static AADepGraphNode *DepGetVal(const DepTy &DT) { return DT.getPointer(); }
+ static AbstractAttribute *DepGetValAA(const DepTy &DT) {
+ return cast<AbstractAttribute>(DT.getPointer());
+ }
+
+ operator AbstractAttribute *() { return cast<AbstractAttribute>(this); }
+
+public:
+ using iterator = mapped_iterator<DepSetTy::iterator, decltype(&DepGetVal)>;
+ using aaiterator =
+ mapped_iterator<DepSetTy::iterator, decltype(&DepGetValAA)>;
+
+ aaiterator begin() { return aaiterator(Deps.begin(), &DepGetValAA); }
+ aaiterator end() { return aaiterator(Deps.end(), &DepGetValAA); }
+ iterator child_begin() { return iterator(Deps.begin(), &DepGetVal); }
+ iterator child_end() { return iterator(Deps.end(), &DepGetVal); }
+
+ void print(raw_ostream &OS) const { print(nullptr, OS); }
+ virtual void print(Attributor *, raw_ostream &OS) const {
+ OS << "AADepNode Impl\n";
+ }
+ DepSetTy &getDeps() { return Deps; }
+
+ friend struct Attributor;
+ friend struct AADepGraph;
+};
+
+/// The data structure for the dependency graph
+///
+/// Note that in this graph if there is an edge from A to B (A -> B),
+/// then it means that B depends on A, and when the state of A is
+/// updated, node B should also be updated
+struct AADepGraph {
+ AADepGraph() = default;
+ ~AADepGraph() = default;
+
+ using DepTy = AADepGraphNode::DepTy;
+ static AADepGraphNode *DepGetVal(const DepTy &DT) { return DT.getPointer(); }
+ using iterator =
+ mapped_iterator<AADepGraphNode::DepSetTy::iterator, decltype(&DepGetVal)>;
+
+ /// There is no root node for the dependency graph. But the SCCIterator
+ /// requires a single entry point, so we maintain a fake("synthetic") root
+ /// node that depends on every node.
+ AADepGraphNode SyntheticRoot;
+ AADepGraphNode *GetEntryNode() { return &SyntheticRoot; }
+
+ iterator begin() { return SyntheticRoot.child_begin(); }
+ iterator end() { return SyntheticRoot.child_end(); }
+
+ void viewGraph();
+
+ /// Dump graph to file
+ void dumpGraph();
+
+ /// Print dependency graph
+ void print();
+};
+
+/// Helper to describe and deal with positions in the LLVM-IR.
+///
+/// A position in the IR is described by an anchor value and an "offset" that
+/// could be the argument number, for call sites and arguments, or an indicator
+/// of the "position kind". The kinds, specified in the Kind enum below, include
+/// the locations in the attribute list, i.a., function scope and return value,
+/// as well as a distinction between call sites and functions. Finally, there
+/// are floating values that do not have a corresponding attribute list
+/// position.
+struct IRPosition {
+ // NOTE: In the future this definition can be changed to support recursive
+ // functions.
+ using CallBaseContext = CallBase;
+
+ /// The positions we distinguish in the IR.
+ enum Kind : char {
+ IRP_INVALID, ///< An invalid position.
+ IRP_FLOAT, ///< A position that is not associated with a spot suitable
+ ///< for attributes. This could be any value or instruction.
+ IRP_RETURNED, ///< An attribute for the function return value.
+ IRP_CALL_SITE_RETURNED, ///< An attribute for a call site return value.
+ IRP_FUNCTION, ///< An attribute for a function (scope).
+ IRP_CALL_SITE, ///< An attribute for a call site (function scope).
+ IRP_ARGUMENT, ///< An attribute for a function argument.
+ IRP_CALL_SITE_ARGUMENT, ///< An attribute for a call site argument.
+ };
+
+ /// Default constructor available to create invalid positions implicitly. All
+ /// other positions need to be created explicitly through the appropriate
+ /// static member function.
+ IRPosition() : Enc(nullptr, ENC_VALUE) { verify(); }
+
+ /// Create a position describing the value of \p V.
+ static const IRPosition value(const Value &V,
+ const CallBaseContext *CBContext = nullptr) {
+ if (auto *Arg = dyn_cast<Argument>(&V))
+ return IRPosition::argument(*Arg, CBContext);
+ if (auto *CB = dyn_cast<CallBase>(&V))
+ return IRPosition::callsite_returned(*CB);
+ return IRPosition(const_cast<Value &>(V), IRP_FLOAT, CBContext);
+ }
+
+ /// Create a position describing the instruction \p I. This is different from
+ /// the value version because call sites are treated as intrusctions rather
+ /// than their return value in this function.
+ static const IRPosition inst(const Instruction &I,
+ const CallBaseContext *CBContext = nullptr) {
+ return IRPosition(const_cast<Instruction &>(I), IRP_FLOAT, CBContext);
+ }
+
+ /// Create a position describing the function scope of \p F.
+ /// \p CBContext is used for call base specific analysis.
+ static const IRPosition function(const Function &F,
+ const CallBaseContext *CBContext = nullptr) {
+ return IRPosition(const_cast<Function &>(F), IRP_FUNCTION, CBContext);
+ }
+
+ /// Create a position describing the returned value of \p F.
+ /// \p CBContext is used for call base specific analysis.
+ static const IRPosition returned(const Function &F,
+ const CallBaseContext *CBContext = nullptr) {
+ return IRPosition(const_cast<Function &>(F), IRP_RETURNED, CBContext);
+ }
+
+ /// Create a position describing the argument \p Arg.
+ /// \p CBContext is used for call base specific analysis.
+ static const IRPosition argument(const Argument &Arg,
+ const CallBaseContext *CBContext = nullptr) {
+ return IRPosition(const_cast<Argument &>(Arg), IRP_ARGUMENT, CBContext);
+ }
+
+ /// Create a position describing the function scope of \p CB.
+ static const IRPosition callsite_function(const CallBase &CB) {
+ return IRPosition(const_cast<CallBase &>(CB), IRP_CALL_SITE);
+ }
+
+ /// Create a position describing the returned value of \p CB.
+ static const IRPosition callsite_returned(const CallBase &CB) {
+ return IRPosition(const_cast<CallBase &>(CB), IRP_CALL_SITE_RETURNED);
+ }
+
+ /// Create a position describing the argument of \p CB at position \p ArgNo.
+ static const IRPosition callsite_argument(const CallBase &CB,
+ unsigned ArgNo) {
+ return IRPosition(const_cast<Use &>(CB.getArgOperandUse(ArgNo)),
+ IRP_CALL_SITE_ARGUMENT);
+ }
+
+ /// Create a position describing the argument of \p ACS at position \p ArgNo.
+ static const IRPosition callsite_argument(AbstractCallSite ACS,
+ unsigned ArgNo) {
+ if (ACS.getNumArgOperands() <= ArgNo)
+ return IRPosition();
+ int CSArgNo = ACS.getCallArgOperandNo(ArgNo);
+ if (CSArgNo >= 0)
+ return IRPosition::callsite_argument(
+ cast<CallBase>(*ACS.getInstruction()), CSArgNo);
+ return IRPosition();
+ }
+
+ /// Create a position with function scope matching the "context" of \p IRP.
+ /// If \p IRP is a call site (see isAnyCallSitePosition()) then the result
+ /// will be a call site position, otherwise the function position of the
+ /// associated function.
+ static const IRPosition
+ function_scope(const IRPosition &IRP,
+ const CallBaseContext *CBContext = nullptr) {
+ if (IRP.isAnyCallSitePosition()) {
+ return IRPosition::callsite_function(
+ cast<CallBase>(IRP.getAnchorValue()));
+ }
+ assert(IRP.getAssociatedFunction());
+ return IRPosition::function(*IRP.getAssociatedFunction(), CBContext);
+ }
+
+ bool operator==(const IRPosition &RHS) const {
+ return Enc == RHS.Enc && RHS.CBContext == CBContext;
+ }
+ bool operator!=(const IRPosition &RHS) const { return !(*this == RHS); }
+
+ /// Return the value this abstract attribute is anchored with.
+ ///
+ /// The anchor value might not be the associated value if the latter is not
+ /// sufficient to determine where arguments will be manifested. This is, so
+ /// far, only the case for call site arguments as the value is not sufficient
+ /// to pinpoint them. Instead, we can use the call site as an anchor.
+ Value &getAnchorValue() const {
+ switch (getEncodingBits()) {
+ case ENC_VALUE:
+ case ENC_RETURNED_VALUE:
+ case ENC_FLOATING_FUNCTION:
+ return *getAsValuePtr();
+ case ENC_CALL_SITE_ARGUMENT_USE:
+ return *(getAsUsePtr()->getUser());
+ default:
+ llvm_unreachable("Unkown encoding!");
+ };
+ }
+
+ /// Return the associated function, if any.
+ Function *getAssociatedFunction() const {
+ if (auto *CB = dyn_cast<CallBase>(&getAnchorValue())) {
+ // We reuse the logic that associates callback calles to arguments of a
+ // call site here to identify the callback callee as the associated
+ // function.
+ if (Argument *Arg = getAssociatedArgument())
+ return Arg->getParent();
+ return dyn_cast_if_present<Function>(
+ CB->getCalledOperand()->stripPointerCasts());
+ }
+ return getAnchorScope();
+ }
+
+ /// Return the associated argument, if any.
+ Argument *getAssociatedArgument() const;
+
+ /// Return true if the position refers to a function interface, that is the
+ /// function scope, the function return, or an argument.
+ bool isFnInterfaceKind() const {
+ switch (getPositionKind()) {
+ case IRPosition::IRP_FUNCTION:
+ case IRPosition::IRP_RETURNED:
+ case IRPosition::IRP_ARGUMENT:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /// Return true if this is a function or call site position.
+ bool isFunctionScope() const {
+ switch (getPositionKind()) {
+ case IRPosition::IRP_CALL_SITE:
+ case IRPosition::IRP_FUNCTION:
+ return true;
+ default:
+ return false;
+ };
+ }
+
+ /// Return the Function surrounding the anchor value.
+ Function *getAnchorScope() const {
+ Value &V = getAnchorValue();
+ if (isa<Function>(V))
+ return &cast<Function>(V);
+ if (isa<Argument>(V))
+ return cast<Argument>(V).getParent();
+ if (isa<Instruction>(V))
+ return cast<Instruction>(V).getFunction();
+ return nullptr;
+ }
+
+ /// Return the context instruction, if any.
+ Instruction *getCtxI() const {
+ Value &V = getAnchorValue();
+ if (auto *I = dyn_cast<Instruction>(&V))
+ return I;
+ if (auto *Arg = dyn_cast<Argument>(&V))
+ if (!Arg->getParent()->isDeclaration())
+ return &Arg->getParent()->getEntryBlock().front();
+ if (auto *F = dyn_cast<Function>(&V))
+ if (!F->isDeclaration())
+ return &(F->getEntryBlock().front());
+ return nullptr;
+ }
+
+ /// Return the value this abstract attribute is associated with.
+ Value &getAssociatedValue() const {
+ if (getCallSiteArgNo() < 0 || isa<Argument>(&getAnchorValue()))
+ return getAnchorValue();
+ assert(isa<CallBase>(&getAnchorValue()) && "Expected a call base!");
+ return *cast<CallBase>(&getAnchorValue())
+ ->getArgOperand(getCallSiteArgNo());
+ }
+
+ /// Return the type this abstract attribute is associated with.
+ Type *getAssociatedType() const {
+ if (getPositionKind() == IRPosition::IRP_RETURNED)
+ return getAssociatedFunction()->getReturnType();
+ return getAssociatedValue().getType();
+ }
+
+ /// Return the callee argument number of the associated value if it is an
+ /// argument or call site argument, otherwise a negative value. In contrast to
+ /// `getCallSiteArgNo` this method will always return the "argument number"
+ /// from the perspective of the callee. This may not the same as the call site
+ /// if this is a callback call.
+ int getCalleeArgNo() const {
+ return getArgNo(/* CallbackCalleeArgIfApplicable */ true);
+ }
+
+ /// Return the call site argument number of the associated value if it is an
+ /// argument or call site argument, otherwise a negative value. In contrast to
+ /// `getCalleArgNo` this method will always return the "operand number" from
+ /// the perspective of the call site. This may not the same as the callee
+ /// perspective if this is a callback call.
+ int getCallSiteArgNo() const {
+ return getArgNo(/* CallbackCalleeArgIfApplicable */ false);
+ }
+
+ /// Return the index in the attribute list for this position.
+ unsigned getAttrIdx() const {
+ switch (getPositionKind()) {
+ case IRPosition::IRP_INVALID:
+ case IRPosition::IRP_FLOAT:
+ break;
+ case IRPosition::IRP_FUNCTION:
+ case IRPosition::IRP_CALL_SITE:
+ return AttributeList::FunctionIndex;
+ case IRPosition::IRP_RETURNED:
+ case IRPosition::IRP_CALL_SITE_RETURNED:
+ return AttributeList::ReturnIndex;
+ case IRPosition::IRP_ARGUMENT:
+ return getCalleeArgNo() + AttributeList::FirstArgIndex;
+ case IRPosition::IRP_CALL_SITE_ARGUMENT:
+ return getCallSiteArgNo() + AttributeList::FirstArgIndex;
+ }
+ llvm_unreachable(
+ "There is no attribute index for a floating or invalid position!");
+ }
+
+ /// Return the value attributes are attached to.
+ Value *getAttrListAnchor() const {
+ if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
+ return CB;
+ return getAssociatedFunction();
+ }
+
+ /// Return the attributes associated with this function or call site scope.
+ AttributeList getAttrList() const {
+ if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
+ return CB->getAttributes();
+ return getAssociatedFunction()->getAttributes();
+ }
+
+ /// Update the attributes associated with this function or call site scope.
+ void setAttrList(const AttributeList &AttrList) const {
+ if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
+ return CB->setAttributes(AttrList);
+ return getAssociatedFunction()->setAttributes(AttrList);
+ }
+
+ /// Return the number of arguments associated with this function or call site
+ /// scope.
+ unsigned getNumArgs() const {
+ assert((getPositionKind() == IRP_CALL_SITE ||
+ getPositionKind() == IRP_FUNCTION) &&
+ "Only valid for function/call site positions!");
+ if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
+ return CB->arg_size();
+ return getAssociatedFunction()->arg_size();
+ }
+
+ /// Return theargument \p ArgNo associated with this function or call site
+ /// scope.
+ Value *getArg(unsigned ArgNo) const {
+ assert((getPositionKind() == IRP_CALL_SITE ||
+ getPositionKind() == IRP_FUNCTION) &&
+ "Only valid for function/call site positions!");
+ if (auto *CB = dyn_cast<CallBase>(&getAnchorValue()))
+ return CB->getArgOperand(ArgNo);
+ return getAssociatedFunction()->getArg(ArgNo);
+ }
+
+ /// Return the associated position kind.
+ Kind getPositionKind() const {
+ char EncodingBits = getEncodingBits();
+ if (EncodingBits == ENC_CALL_SITE_ARGUMENT_USE)
+ return IRP_CALL_SITE_ARGUMENT;
+ if (EncodingBits == ENC_FLOATING_FUNCTION)
+ return IRP_FLOAT;
+
+ Value *V = getAsValuePtr();
+ if (!V)
+ return IRP_INVALID;
+ if (isa<Argument>(V))
+ return IRP_ARGUMENT;
+ if (isa<Function>(V))
+ return isReturnPosition(EncodingBits) ? IRP_RETURNED : IRP_FUNCTION;
+ if (isa<CallBase>(V))
+ return isReturnPosition(EncodingBits) ? IRP_CALL_SITE_RETURNED
+ : IRP_CALL_SITE;
+ return IRP_FLOAT;
+ }
+
+ bool isAnyCallSitePosition() const {
+ switch (getPositionKind()) {
+ case IRPosition::IRP_CALL_SITE:
+ case IRPosition::IRP_CALL_SITE_RETURNED:
+ case IRPosition::IRP_CALL_SITE_ARGUMENT:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /// Return true if the position is an argument or call site argument.
+ bool isArgumentPosition() const {
+ switch (getPositionKind()) {
+ case IRPosition::IRP_ARGUMENT:
+ case IRPosition::IRP_CALL_SITE_ARGUMENT:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /// Return the same position without the call base context.
+ IRPosition stripCallBaseContext() const {
+ IRPosition Result = *this;
+ Result.CBContext = nullptr;
+ return Result;
+ }
+
+ /// Get the call base context from the position.
+ const CallBaseContext *getCallBaseContext() const { return CBContext; }
+
+ /// Check if the position has any call base context.
+ bool hasCallBaseContext() const { return CBContext != nullptr; }
+
+ /// Special DenseMap key values.
+ ///
+ ///{
+ static const IRPosition EmptyKey;
+ static const IRPosition TombstoneKey;
+ ///}
+
+ /// Conversion into a void * to allow reuse of pointer hashing.
+ operator void *() const { return Enc.getOpaqueValue(); }
+
+private:
+ /// Private constructor for special values only!
+ explicit IRPosition(void *Ptr, const CallBaseContext *CBContext = nullptr)
+ : CBContext(CBContext) {
+ Enc.setFromOpaqueValue(Ptr);
+ }
+
+ /// IRPosition anchored at \p AnchorVal with kind/argument numbet \p PK.
+ explicit IRPosition(Value &AnchorVal, Kind PK,
+ const CallBaseContext *CBContext = nullptr)
+ : CBContext(CBContext) {
+ switch (PK) {
+ case IRPosition::IRP_INVALID:
+ llvm_unreachable("Cannot create invalid IRP with an anchor value!");
+ break;
+ case IRPosition::IRP_FLOAT:
+ // Special case for floating functions.
+ if (isa<Function>(AnchorVal) || isa<CallBase>(AnchorVal))
+ Enc = {&AnchorVal, ENC_FLOATING_FUNCTION};
+ else
+ Enc = {&AnchorVal, ENC_VALUE};
+ break;
+ case IRPosition::IRP_FUNCTION:
+ case IRPosition::IRP_CALL_SITE:
+ Enc = {&AnchorVal, ENC_VALUE};
+ break;
+ case IRPosition::IRP_RETURNED:
+ case IRPosition::IRP_CALL_SITE_RETURNED:
+ Enc = {&AnchorVal, ENC_RETURNED_VALUE};
+ break;
+ case IRPosition::IRP_ARGUMENT:
+ Enc = {&AnchorVal, ENC_VALUE};
+ break;
+ case IRPosition::IRP_CALL_SITE_ARGUMENT:
+ llvm_unreachable(
+ "Cannot create call site argument IRP with an anchor value!");
+ break;
+ }
+ verify();
+ }
+
+ /// Return the callee argument number of the associated value if it is an
+ /// argument or call site argument. See also `getCalleeArgNo` and
+ /// `getCallSiteArgNo`.
+ int getArgNo(bool CallbackCalleeArgIfApplicable) const {
+ if (CallbackCalleeArgIfApplicable)
+ if (Argument *Arg = getAssociatedArgument())
+ return Arg->getArgNo();
+ switch (getPositionKind()) {
+ case IRPosition::IRP_ARGUMENT:
+ return cast<Argument>(getAsValuePtr())->getArgNo();
+ case IRPosition::IRP_CALL_SITE_ARGUMENT: {
+ Use &U = *getAsUsePtr();
+ return cast<CallBase>(U.getUser())->getArgOperandNo(&U);
+ }
+ default:
+ return -1;
+ }
+ }
+
+ /// IRPosition for the use \p U. The position kind \p PK needs to be
+ /// IRP_CALL_SITE_ARGUMENT, the anchor value is the user, the associated value
+ /// the used value.
+ explicit IRPosition(Use &U, Kind PK) {
+ assert(PK == IRP_CALL_SITE_ARGUMENT &&
+ "Use constructor is for call site arguments only!");
+ Enc = {&U, ENC_CALL_SITE_ARGUMENT_USE};
+ verify();
+ }
+
+ /// Verify internal invariants.
+ void verify();
+
+ /// Return the underlying pointer as Value *, valid for all positions but
+ /// IRP_CALL_SITE_ARGUMENT.
+ Value *getAsValuePtr() const {
+ assert(getEncodingBits() != ENC_CALL_SITE_ARGUMENT_USE &&
+ "Not a value pointer!");
+ return reinterpret_cast<Value *>(Enc.getPointer());
+ }
+
+ /// Return the underlying pointer as Use *, valid only for
+ /// IRP_CALL_SITE_ARGUMENT positions.
+ Use *getAsUsePtr() const {
+ assert(getEncodingBits() == ENC_CALL_SITE_ARGUMENT_USE &&
+ "Not a value pointer!");
+ return reinterpret_cast<Use *>(Enc.getPointer());
+ }
+
+ /// Return true if \p EncodingBits describe a returned or call site returned
+ /// position.
+ static bool isReturnPosition(char EncodingBits) {
+ return EncodingBits == ENC_RETURNED_VALUE;
+ }
+
+ /// Return true if the encoding bits describe a returned or call site returned
+ /// position.
+ bool isReturnPosition() const { return isReturnPosition(getEncodingBits()); }
+
+ /// The encoding of the IRPosition is a combination of a pointer and two
+ /// encoding bits. The values of the encoding bits are defined in the enum
+ /// below. The pointer is either a Value* (for the first three encoding bit
+ /// combinations) or Use* (for ENC_CALL_SITE_ARGUMENT_USE).
+ ///
+ ///{
+ enum {
+ ENC_VALUE = 0b00,
+ ENC_RETURNED_VALUE = 0b01,
+ ENC_FLOATING_FUNCTION = 0b10,
+ ENC_CALL_SITE_ARGUMENT_USE = 0b11,
+ };
+
+ // Reserve the maximal amount of bits so there is no need to mask out the
+ // remaining ones. We will not encode anything else in the pointer anyway.
+ static constexpr int NumEncodingBits =
+ PointerLikeTypeTraits<void *>::NumLowBitsAvailable;
+ static_assert(NumEncodingBits >= 2, "At least two bits are required!");
+
+ /// The pointer with the encoding bits.
+ PointerIntPair<void *, NumEncodingBits, char> Enc;
+ ///}
+
+ /// Call base context. Used for callsite specific analysis.
+ const CallBaseContext *CBContext = nullptr;
+
+ /// Return the encoding bits.
+ char getEncodingBits() const { return Enc.getInt(); }
+};
+
+/// Helper that allows IRPosition as a key in a DenseMap.
+template <> struct DenseMapInfo<IRPosition> {
+ static inline IRPosition getEmptyKey() { return IRPosition::EmptyKey; }
+ static inline IRPosition getTombstoneKey() {
+ return IRPosition::TombstoneKey;
+ }
+ static unsigned getHashValue(const IRPosition &IRP) {
+ return (DenseMapInfo<void *>::getHashValue(IRP) << 4) ^
+ (DenseMapInfo<Value *>::getHashValue(IRP.getCallBaseContext()));
+ }
+
+ static bool isEqual(const IRPosition &a, const IRPosition &b) {
+ return a == b;
+ }
+};
+
+/// A visitor class for IR positions.
+///
+/// Given a position P, the SubsumingPositionIterator allows to visit "subsuming
+/// positions" wrt. attributes/information. Thus, if a piece of information
+/// holds for a subsuming position, it also holds for the position P.
+///
+/// The subsuming positions always include the initial position and then,
+/// depending on the position kind, additionally the following ones:
+/// - for IRP_RETURNED:
+/// - the function (IRP_FUNCTION)
+/// - for IRP_ARGUMENT:
+/// - the function (IRP_FUNCTION)
+/// - for IRP_CALL_SITE:
+/// - the callee (IRP_FUNCTION), if known
+/// - for IRP_CALL_SITE_RETURNED:
+/// - the callee (IRP_RETURNED), if known
+/// - the call site (IRP_FUNCTION)
+/// - the callee (IRP_FUNCTION), if known
+/// - for IRP_CALL_SITE_ARGUMENT:
+/// - the argument of the callee (IRP_ARGUMENT), if known
+/// - the callee (IRP_FUNCTION), if known
+/// - the position the call site argument is associated with if it is not
+/// anchored to the call site, e.g., if it is an argument then the argument
+/// (IRP_ARGUMENT)
+class SubsumingPositionIterator {
+ SmallVector<IRPosition, 4> IRPositions;
+ using iterator = decltype(IRPositions)::iterator;
+
+public:
+ SubsumingPositionIterator(const IRPosition &IRP);
+ iterator begin() { return IRPositions.begin(); }
+ iterator end() { return IRPositions.end(); }
+};
+
+/// Wrapper for FunctionAnalysisManager.
+struct AnalysisGetter {
+ // The client may be running the old pass manager, in which case, we need to
+ // map the requested Analysis to its equivalent wrapper in the old pass
+ // manager. The scheme implemented here does not require every Analysis to be
+ // updated. Only those new analyses that the client cares about in the old
+ // pass manager need to expose a LegacyWrapper type, and that wrapper should
+ // support a getResult() method that matches the new Analysis.
+ //
+ // We need SFINAE to check for the LegacyWrapper, but function templates don't
+ // allow partial specialization, which is needed in this case. So instead, we
+ // use a constexpr bool to perform the SFINAE, and then use this information
+ // inside the function template.
+ template <typename, typename = void>
+ static constexpr bool HasLegacyWrapper = false;
+
+ template <typename Analysis>
+ typename Analysis::Result *getAnalysis(const Function &F,
+ bool RequestCachedOnly = false) {
+ if (!LegacyPass && !FAM)
+ return nullptr;
+ if (FAM) {
+ if (CachedOnly || RequestCachedOnly)
+ return FAM->getCachedResult<Analysis>(const_cast<Function &>(F));
+ return &FAM->getResult<Analysis>(const_cast<Function &>(F));
+ }
+ if constexpr (HasLegacyWrapper<Analysis>) {
+ if (!CachedOnly && !RequestCachedOnly)
+ return &LegacyPass
+ ->getAnalysis<typename Analysis::LegacyWrapper>(
+ const_cast<Function &>(F))
+ .getResult();
+ if (auto *P =
+ LegacyPass
+ ->getAnalysisIfAvailable<typename Analysis::LegacyWrapper>())
+ return &P->getResult();
+ }
+ return nullptr;
+ }
+
+ /// Invalidates the analyses. Valid only when using the new pass manager.
+ void invalidateAnalyses() {
+ assert(FAM && "Can only be used from the new PM!");
+ FAM->clear();
+ }
+
+ AnalysisGetter(FunctionAnalysisManager &FAM, bool CachedOnly = false)
+ : FAM(&FAM), CachedOnly(CachedOnly) {}
+ AnalysisGetter(Pass *P, bool CachedOnly = false)
+ : LegacyPass(P), CachedOnly(CachedOnly) {}
+ AnalysisGetter() = default;
+
+private:
+ FunctionAnalysisManager *FAM = nullptr;
+ Pass *LegacyPass = nullptr;
+
+ /// If \p CachedOnly is true, no pass is created, just existing results are
+ /// used. Also available per request.
+ bool CachedOnly = false;
+};
+
+template <typename Analysis>
+constexpr bool AnalysisGetter::HasLegacyWrapper<
+ Analysis, std::void_t<typename Analysis::LegacyWrapper>> = true;
+
+/// Data structure to hold cached (LLVM-IR) information.
+///
+/// All attributes are given an InformationCache object at creation time to
+/// avoid inspection of the IR by all of them individually. This default
+/// InformationCache will hold information required by 'default' attributes,
+/// thus the ones deduced when Attributor::identifyDefaultAbstractAttributes(..)
+/// is called.
+///
+/// If custom abstract attributes, registered manually through
+/// Attributor::registerAA(...), need more information, especially if it is not
+/// reusable, it is advised to inherit from the InformationCache and cast the
+/// instance down in the abstract attributes.
+struct InformationCache {
+ InformationCache(const Module &M, AnalysisGetter &AG,
+ BumpPtrAllocator &Allocator, SetVector<Function *> *CGSCC,
+ bool UseExplorer = true)
+ : CGSCC(CGSCC), DL(M.getDataLayout()), Allocator(Allocator), AG(AG),
+ TargetTriple(M.getTargetTriple()) {
+ if (UseExplorer)
+ Explorer = new (Allocator) MustBeExecutedContextExplorer(
+ /* ExploreInterBlock */ true, /* ExploreCFGForward */ true,
+ /* ExploreCFGBackward */ true,
+ /* LIGetter */
+ [&](const Function &F) { return AG.getAnalysis<LoopAnalysis>(F); },
+ /* DTGetter */
+ [&](const Function &F) {
+ return AG.getAnalysis<DominatorTreeAnalysis>(F);
+ },
+ /* PDTGetter */
+ [&](const Function &F) {
+ return AG.getAnalysis<PostDominatorTreeAnalysis>(F);
+ });
+ }
+
+ ~InformationCache() {
+ // The FunctionInfo objects are allocated via a BumpPtrAllocator, we call
+ // the destructor manually.
+ for (auto &It : FuncInfoMap)
+ It.getSecond()->~FunctionInfo();
+ // Same is true for the instruction exclusions sets.
+ using AA::InstExclusionSetTy;
+ for (auto *BES : BESets)
+ BES->~InstExclusionSetTy();
+ if (Explorer)
+ Explorer->~MustBeExecutedContextExplorer();
+ }
+
+ /// Apply \p CB to all uses of \p F. If \p LookThroughConstantExprUses is
+ /// true, constant expression users are not given to \p CB but their uses are
+ /// traversed transitively.
+ template <typename CBTy>
+ static void foreachUse(Function &F, CBTy CB,
+ bool LookThroughConstantExprUses = true) {
+ SmallVector<Use *, 8> Worklist(make_pointer_range(F.uses()));
+
+ for (unsigned Idx = 0; Idx < Worklist.size(); ++Idx) {
+ Use &U = *Worklist[Idx];
+
+ // Allow use in constant bitcasts and simply look through them.
+ if (LookThroughConstantExprUses && isa<ConstantExpr>(U.getUser())) {
+ for (Use &CEU : cast<ConstantExpr>(U.getUser())->uses())
+ Worklist.push_back(&CEU);
+ continue;
+ }
+
+ CB(U);
+ }
+ }
+
+ /// The CG-SCC the pass is run on, or nullptr if it is a module pass.
+ const SetVector<Function *> *const CGSCC = nullptr;
+
+ /// A vector type to hold instructions.
+ using InstructionVectorTy = SmallVector<Instruction *, 8>;
+
+ /// A map type from opcodes to instructions with this opcode.
+ using OpcodeInstMapTy = DenseMap<unsigned, InstructionVectorTy *>;
+
+ /// Return the map that relates "interesting" opcodes with all instructions
+ /// with that opcode in \p F.
+ OpcodeInstMapTy &getOpcodeInstMapForFunction(const Function &F) {
+ return getFunctionInfo(F).OpcodeInstMap;
+ }
+
+ /// Return the instructions in \p F that may read or write memory.
+ InstructionVectorTy &getReadOrWriteInstsForFunction(const Function &F) {
+ return getFunctionInfo(F).RWInsts;
+ }
+
+ /// Return MustBeExecutedContextExplorer
+ MustBeExecutedContextExplorer *getMustBeExecutedContextExplorer() {
+ return Explorer;
+ }
+
+ /// Return TargetLibraryInfo for function \p F.
+ TargetLibraryInfo *getTargetLibraryInfoForFunction(const Function &F) {
+ return AG.getAnalysis<TargetLibraryAnalysis>(F);
+ }
+
+ /// Return true if \p Arg is involved in a must-tail call, thus the argument
+ /// of the caller or callee.
+ bool isInvolvedInMustTailCall(const Argument &Arg) {
+ FunctionInfo &FI = getFunctionInfo(*Arg.getParent());
+ return FI.CalledViaMustTail || FI.ContainsMustTailCall;
+ }
+
+ bool isOnlyUsedByAssume(const Instruction &I) const {
+ return AssumeOnlyValues.contains(&I);
+ }
+
+ /// Invalidates the cached analyses. Valid only when using the new pass
+ /// manager.
+ void invalidateAnalyses() { AG.invalidateAnalyses(); }
+
+ /// Return the analysis result from a pass \p AP for function \p F.
+ template <typename AP>
+ typename AP::Result *getAnalysisResultForFunction(const Function &F,
+ bool CachedOnly = false) {
+ return AG.getAnalysis<AP>(F, CachedOnly);
+ }
+
+ /// Return datalayout used in the module.
+ const DataLayout &getDL() { return DL; }
+
+ /// Return the map conaining all the knowledge we have from `llvm.assume`s.
+ const RetainedKnowledgeMap &getKnowledgeMap() const { return KnowledgeMap; }
+
+ /// Given \p BES, return a uniqued version.
+ const AA::InstExclusionSetTy *
+ getOrCreateUniqueBlockExecutionSet(const AA::InstExclusionSetTy *BES) {
+ auto It = BESets.find(BES);
+ if (It != BESets.end())
+ return *It;
+ auto *UniqueBES = new (Allocator) AA::InstExclusionSetTy(*BES);
+ bool Success = BESets.insert(UniqueBES).second;
+ (void)Success;
+ assert(Success && "Expected only new entries to be added");
+ return UniqueBES;
+ }
+
+ /// Return true if the stack (llvm::Alloca) can be accessed by other threads.
+ bool stackIsAccessibleByOtherThreads() { return !targetIsGPU(); }
+
+ /// Return true if the target is a GPU.
+ bool targetIsGPU() {
+ return TargetTriple.isAMDGPU() || TargetTriple.isNVPTX();
+ }
+
+ /// Return all functions that might be called indirectly, only valid for
+ /// closed world modules (see isClosedWorldModule).
+ const ArrayRef<Function *>
+ getIndirectlyCallableFunctions(Attributor &A) const;
+
+private:
+ struct FunctionInfo {
+ ~FunctionInfo();
+
+ /// A nested map that remembers all instructions in a function with a
+ /// certain instruction opcode (Instruction::getOpcode()).
+ OpcodeInstMapTy OpcodeInstMap;
+
+ /// A map from functions to their instructions that may read or write
+ /// memory.
+ InstructionVectorTy RWInsts;
+
+ /// Function is called by a `musttail` call.
+ bool CalledViaMustTail;
+
+ /// Function contains a `musttail` call.
+ bool ContainsMustTailCall;
+ };
+
+ /// A map type from functions to informatio about it.
+ DenseMap<const Function *, FunctionInfo *> FuncInfoMap;
+
+ /// Return information about the function \p F, potentially by creating it.
+ FunctionInfo &getFunctionInfo(const Function &F) {
+ FunctionInfo *&FI = FuncInfoMap[&F];
+ if (!FI) {
+ FI = new (Allocator) FunctionInfo();
+ initializeInformationCache(F, *FI);
+ }
+ return *FI;
+ }
+
+ /// Vector of functions that might be callable indirectly, i.a., via a
+ /// function pointer.
+ SmallVector<Function *> IndirectlyCallableFunctions;
+
+ /// Initialize the function information cache \p FI for the function \p F.
+ ///
+ /// This method needs to be called for all function that might be looked at
+ /// through the information cache interface *prior* to looking at them.
+ void initializeInformationCache(const Function &F, FunctionInfo &FI);
+
+ /// The datalayout used in the module.
+ const DataLayout &DL;
+
+ /// The allocator used to allocate memory, e.g. for `FunctionInfo`s.
+ BumpPtrAllocator &Allocator;
+
+ /// MustBeExecutedContextExplorer
+ MustBeExecutedContextExplorer *Explorer = nullptr;
+
+ /// A map with knowledge retained in `llvm.assume` instructions.
+ RetainedKnowledgeMap KnowledgeMap;
+
+ /// A container for all instructions that are only used by `llvm.assume`.
+ SetVector<const Instruction *> AssumeOnlyValues;
+
+ /// Cache for block sets to allow reuse.
+ DenseSet<const AA::InstExclusionSetTy *> BESets;
+
+ /// Getters for analysis.
+ AnalysisGetter &AG;
+
+ /// Set of inlineable functions
+ SmallPtrSet<const Function *, 8> InlineableFunctions;
+
+ /// The triple describing the target machine.
+ Triple TargetTriple;
+
+ /// Give the Attributor access to the members so
+ /// Attributor::identifyDefaultAbstractAttributes(...) can initialize them.
+ friend struct Attributor;
+};
+
+/// Configuration for the Attributor.
+struct AttributorConfig {
+
+ AttributorConfig(CallGraphUpdater &CGUpdater) : CGUpdater(CGUpdater) {}
+
+ /// Is the user of the Attributor a module pass or not. This determines what
+ /// IR we can look at and modify. If it is a module pass we might deduce facts
+ /// outside the initial function set and modify functions outside that set,
+ /// but only as part of the optimization of the functions in the initial
+ /// function set. For CGSCC passes we can look at the IR of the module slice
+ /// but never run any deduction, or perform any modification, outside the
+ /// initial function set (which we assume is the SCC).
+ bool IsModulePass = true;
+
+ /// Flag to determine if we can delete functions or keep dead ones around.
+ bool DeleteFns = true;
+
+ /// Flag to determine if we rewrite function signatures.
+ bool RewriteSignatures = true;
+
+ /// Flag to determine if we want to initialize all default AAs for an internal
+ /// function marked live. See also: InitializationCallback>
+ bool DefaultInitializeLiveInternals = true;
+
+ /// Flag to determine if we should skip all liveness checks early on.
+ bool UseLiveness = true;
+
+ /// Flag to indicate if the entire world is contained in this module, that
+ /// is, no outside functions exist.
+ bool IsClosedWorldModule = false;
+
+ /// Callback function to be invoked on internal functions marked live.
+ std::function<void(Attributor &A, const Function &F)> InitializationCallback =
+ nullptr;
+
+ /// Callback function to determine if an indirect call targets should be made
+ /// direct call targets (with an if-cascade).
+ std::function<bool(Attributor &A, const AbstractAttribute &AA, CallBase &CB,
+ Function &AssummedCallee)>
+ IndirectCalleeSpecializationCallback = nullptr;
+
+ /// Helper to update an underlying call graph and to delete functions.
+ CallGraphUpdater &CGUpdater;
+
+ /// If not null, a set limiting the attribute opportunities.
+ DenseSet<const char *> *Allowed = nullptr;
+
+ /// Maximum number of iterations to run until fixpoint.
+ std::optional<unsigned> MaxFixpointIterations;
+
+ /// A callback function that returns an ORE object from a Function pointer.
+ ///{
+ using OptimizationRemarkGetter =
+ function_ref<OptimizationRemarkEmitter &(Function *)>;
+ OptimizationRemarkGetter OREGetter = nullptr;
+ ///}
+
+ /// The name of the pass running the attributor, used to emit remarks.
+ const char *PassName = nullptr;
+
+ using IPOAmendableCBTy = function_ref<bool(const Function &F)>;
+ IPOAmendableCBTy IPOAmendableCB;
+};
+
+/// A debug counter to limit the number of AAs created.
+DEBUG_COUNTER(NumAbstractAttributes, "num-abstract-attributes",
+ "How many AAs should be initialized");
+
+/// The fixpoint analysis framework that orchestrates the attribute deduction.
+///
+/// The Attributor provides a general abstract analysis framework (guided
+/// fixpoint iteration) as well as helper functions for the deduction of
+/// (LLVM-IR) attributes. However, also other code properties can be deduced,
+/// propagated, and ultimately manifested through the Attributor framework. This
+/// is particularly useful if these properties interact with attributes and a
+/// co-scheduled deduction allows to improve the solution. Even if not, thus if
+/// attributes/properties are completely isolated, they should use the
+/// Attributor framework to reduce the number of fixpoint iteration frameworks
+/// in the code base. Note that the Attributor design makes sure that isolated
+/// attributes are not impacted, in any way, by others derived at the same time
+/// if there is no cross-reasoning performed.
+///
+/// The public facing interface of the Attributor is kept simple and basically
+/// allows abstract attributes to one thing, query abstract attributes
+/// in-flight. There are two reasons to do this:
+/// a) The optimistic state of one abstract attribute can justify an
+/// optimistic state of another, allowing to framework to end up with an
+/// optimistic (=best possible) fixpoint instead of one based solely on
+/// information in the IR.
+/// b) This avoids reimplementing various kinds of lookups, e.g., to check
+/// for existing IR attributes, in favor of a single lookups interface
+/// provided by an abstract attribute subclass.
+///
+/// NOTE: The mechanics of adding a new "concrete" abstract attribute are
+/// described in the file comment.
+struct Attributor {
+
+ /// Constructor
+ ///
+ /// \param Functions The set of functions we are deriving attributes for.
+ /// \param InfoCache Cache to hold various information accessible for
+ /// the abstract attributes.
+ /// \param Configuration The Attributor configuration which determines what
+ /// generic features to use.
+ Attributor(SetVector<Function *> &Functions, InformationCache &InfoCache,
+ AttributorConfig Configuration);
+
+ ~Attributor();
+
+ /// Run the analyses until a fixpoint is reached or enforced (timeout).
+ ///
+ /// The attributes registered with this Attributor can be used after as long
+ /// as the Attributor is not destroyed (it owns the attributes now).
+ ///
+ /// \Returns CHANGED if the IR was changed, otherwise UNCHANGED.
+ ChangeStatus run();
+
+ /// Lookup an abstract attribute of type \p AAType at position \p IRP. While
+ /// no abstract attribute is found equivalent positions are checked, see
+ /// SubsumingPositionIterator. Thus, the returned abstract attribute
+ /// might be anchored at a different position, e.g., the callee if \p IRP is a
+ /// call base.
+ ///
+ /// This method is the only (supported) way an abstract attribute can retrieve
+ /// information from another abstract attribute. As an example, take an
+ /// abstract attribute that determines the memory access behavior for a
+ /// argument (readnone, readonly, ...). It should use `getAAFor` to get the
+ /// most optimistic information for other abstract attributes in-flight, e.g.
+ /// the one reasoning about the "captured" state for the argument or the one
+ /// reasoning on the memory access behavior of the function as a whole.
+ ///
+ /// If the DepClass enum is set to `DepClassTy::None` the dependence from
+ /// \p QueryingAA to the return abstract attribute is not automatically
+ /// recorded. This should only be used if the caller will record the
+ /// dependence explicitly if necessary, thus if it the returned abstract
+ /// attribute is used for reasoning. To record the dependences explicitly use
+ /// the `Attributor::recordDependence` method.
+ template <typename AAType>
+ const AAType *getAAFor(const AbstractAttribute &QueryingAA,
+ const IRPosition &IRP, DepClassTy DepClass) {
+ return getOrCreateAAFor<AAType>(IRP, &QueryingAA, DepClass,
+ /* ForceUpdate */ false);
+ }
+
+ /// The version of getAAFor that allows to omit a querying abstract
+ /// attribute. Using this after Attributor started running is restricted to
+ /// only the Attributor itself. Initial seeding of AAs can be done via this
+ /// function.
+ /// NOTE: ForceUpdate is ignored in any stage other than the update stage.
+ template <typename AAType>
+ const AAType *getOrCreateAAFor(IRPosition IRP,
+ const AbstractAttribute *QueryingAA,
+ DepClassTy DepClass, bool ForceUpdate = false,
+ bool UpdateAfterInit = true) {
+ if (!shouldPropagateCallBaseContext(IRP))
+ IRP = IRP.stripCallBaseContext();
+
+ if (AAType *AAPtr = lookupAAFor<AAType>(IRP, QueryingAA, DepClass,
+ /* AllowInvalidState */ true)) {
+ if (ForceUpdate && Phase == AttributorPhase::UPDATE)
+ updateAA(*AAPtr);
+ return AAPtr;
+ }
+
+ bool ShouldUpdateAA;
+ if (!shouldInitialize<AAType>(IRP, ShouldUpdateAA))
+ return nullptr;
+
+ if (!DebugCounter::shouldExecute(NumAbstractAttributes))
+ return nullptr;
+
+ // No matching attribute found, create one.
+ // Use the static create method.
+ auto &AA = AAType::createForPosition(IRP, *this);
+
+ // Always register a new attribute to make sure we clean up the allocated
+ // memory properly.
+ registerAA(AA);
+
+ // If we are currenty seeding attributes, enforce seeding rules.
+ if (Phase == AttributorPhase::SEEDING && !shouldSeedAttribute(AA)) {
+ AA.getState().indicatePessimisticFixpoint();
+ return &AA;
+ }
+
+ // Bootstrap the new attribute with an initial update to propagate
+ // information, e.g., function -> call site.
+ {
+ TimeTraceScope TimeScope("initialize", [&]() {
+ return AA.getName() +
+ std::to_string(AA.getIRPosition().getPositionKind());
+ });
+ ++InitializationChainLength;
+ AA.initialize(*this);
+ --InitializationChainLength;
+ }
+
+ if (!ShouldUpdateAA) {
+ AA.getState().indicatePessimisticFixpoint();
+ return &AA;
+ }
+
+ // Allow seeded attributes to declare dependencies.
+ // Remember the seeding state.
+ if (UpdateAfterInit) {
+ AttributorPhase OldPhase = Phase;
+ Phase = AttributorPhase::UPDATE;
+
+ updateAA(AA);
+
+ Phase = OldPhase;
+ }
+
+ if (QueryingAA && AA.getState().isValidState())
+ recordDependence(AA, const_cast<AbstractAttribute &>(*QueryingAA),
+ DepClass);
+ return &AA;
+ }
+
+ template <typename AAType>
+ const AAType *getOrCreateAAFor(const IRPosition &IRP) {
+ return getOrCreateAAFor<AAType>(IRP, /* QueryingAA */ nullptr,
+ DepClassTy::NONE);
+ }
+
+ /// Return the attribute of \p AAType for \p IRP if existing and valid. This
+ /// also allows non-AA users lookup.
+ template <typename AAType>
+ AAType *lookupAAFor(const IRPosition &IRP,
+ const AbstractAttribute *QueryingAA = nullptr,
+ DepClassTy DepClass = DepClassTy::OPTIONAL,
+ bool AllowInvalidState = false) {
+ static_assert(std::is_base_of<AbstractAttribute, AAType>::value,
+ "Cannot query an attribute with a type not derived from "
+ "'AbstractAttribute'!");
+ // Lookup the abstract attribute of type AAType. If found, return it after
+ // registering a dependence of QueryingAA on the one returned attribute.
+ AbstractAttribute *AAPtr = AAMap.lookup({&AAType::ID, IRP});
+ if (!AAPtr)
+ return nullptr;
+
+ AAType *AA = static_cast<AAType *>(AAPtr);
+
+ // Do not register a dependence on an attribute with an invalid state.
+ if (DepClass != DepClassTy::NONE && QueryingAA &&
+ AA->getState().isValidState())
+ recordDependence(*AA, const_cast<AbstractAttribute &>(*QueryingAA),
+ DepClass);
+
+ // Return nullptr if this attribute has an invalid state.
+ if (!AllowInvalidState && !AA->getState().isValidState())
+ return nullptr;
+ return AA;
+ }
+
+ /// Allows a query AA to request an update if a new query was received.
+ void registerForUpdate(AbstractAttribute &AA);
+
+ /// Explicitly record a dependence from \p FromAA to \p ToAA, that is if
+ /// \p FromAA changes \p ToAA should be updated as well.
+ ///
+ /// This method should be used in conjunction with the `getAAFor` method and
+ /// with the DepClass enum passed to the method set to None. This can
+ /// be beneficial to avoid false dependences but it requires the users of
+ /// `getAAFor` to explicitly record true dependences through this method.
+ /// The \p DepClass flag indicates if the dependence is striclty necessary.
+ /// That means for required dependences, if \p FromAA changes to an invalid
+ /// state, \p ToAA can be moved to a pessimistic fixpoint because it required
+ /// information from \p FromAA but none are available anymore.
+ void recordDependence(const AbstractAttribute &FromAA,
+ const AbstractAttribute &ToAA, DepClassTy DepClass);
+
+ /// Introduce a new abstract attribute into the fixpoint analysis.
+ ///
+ /// Note that ownership of the attribute is given to the Attributor. It will
+ /// invoke delete for the Attributor on destruction of the Attributor.
+ ///
+ /// Attributes are identified by their IR position (AAType::getIRPosition())
+ /// and the address of their static member (see AAType::ID).
+ template <typename AAType> AAType ®isterAA(AAType &AA) {
+ static_assert(std::is_base_of<AbstractAttribute, AAType>::value,
+ "Cannot register an attribute with a type not derived from "
+ "'AbstractAttribute'!");
+ // Put the attribute in the lookup map structure and the container we use to
+ // keep track of all attributes.
+ const IRPosition &IRP = AA.getIRPosition();
+ AbstractAttribute *&AAPtr = AAMap[{&AAType::ID, IRP}];
+
+ assert(!AAPtr && "Attribute already in map!");
+ AAPtr = &AA;
+
+ // Register AA with the synthetic root only before the manifest stage.
+ if (Phase == AttributorPhase::SEEDING || Phase == AttributorPhase::UPDATE)
+ DG.SyntheticRoot.Deps.insert(
+ AADepGraphNode::DepTy(&AA, unsigned(DepClassTy::REQUIRED)));
+
+ return AA;
+ }
+
+ /// Return the internal information cache.
+ InformationCache &getInfoCache() { return InfoCache; }
+
+ /// Return true if this is a module pass, false otherwise.
+ bool isModulePass() const { return Configuration.IsModulePass; }
+
+ /// Return true if we should specialize the call site \b CB for the potential
+ /// callee \p Fn.
+ bool shouldSpecializeCallSiteForCallee(const AbstractAttribute &AA,
+ CallBase &CB, Function &Callee) {
+ return Configuration.IndirectCalleeSpecializationCallback
+ ? Configuration.IndirectCalleeSpecializationCallback(*this, AA,
+ CB, Callee)
+ : true;
+ }
+
+ /// Return true if the module contains the whole world, thus, no outside
+ /// functions exist.
+ bool isClosedWorldModule() const;
+
+ /// Return true if we derive attributes for \p Fn
+ bool isRunOn(Function &Fn) const { return isRunOn(&Fn); }
+ bool isRunOn(Function *Fn) const {
+ return Functions.empty() || Functions.count(Fn);
+ }
+
+ template <typename AAType> bool shouldUpdateAA(const IRPosition &IRP) {
+ // If this is queried in the manifest stage, we force the AA to indicate
+ // pessimistic fixpoint immediately.
+ if (Phase == AttributorPhase::MANIFEST || Phase == AttributorPhase::CLEANUP)
+ return false;
+
+ Function *AssociatedFn = IRP.getAssociatedFunction();
+
+ if (IRP.isAnyCallSitePosition()) {
+ // Check if we require a callee but there is none.
+ if (!AssociatedFn && AAType::requiresCalleeForCallBase())
+ return false;
+
+ // Check if we require non-asm but it is inline asm.
+ if (AAType::requiresNonAsmForCallBase() &&
+ cast<CallBase>(IRP.getAnchorValue()).isInlineAsm())
+ return false;
+ }
+
+ // Check if we require a calles but we can't see all.
+ if (AAType::requiresCallersForArgOrFunction())
+ if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION ||
+ IRP.getPositionKind() == IRPosition::IRP_ARGUMENT)
+ if (!AssociatedFn->hasLocalLinkage())
+ return false;
+
+ if (!AAType::isValidIRPositionForUpdate(*this, IRP))
+ return false;
+
+ // We update only AAs associated with functions in the Functions set or
+ // call sites of them.
+ return (!AssociatedFn || isModulePass() || isRunOn(AssociatedFn) ||
+ isRunOn(IRP.getAnchorScope()));
+ }
+
+ template <typename AAType>
+ bool shouldInitialize(const IRPosition &IRP, bool &ShouldUpdateAA) {
+ if (!AAType::isValidIRPositionForInit(*this, IRP))
+ return false;
+
+ if (Configuration.Allowed && !Configuration.Allowed->count(&AAType::ID))
+ return false;
+
+ // For now we skip anything in naked and optnone functions.
+ const Function *AnchorFn = IRP.getAnchorScope();
+ if (AnchorFn && (AnchorFn->hasFnAttribute(Attribute::Naked) ||
+ AnchorFn->hasFnAttribute(Attribute::OptimizeNone)))
+ return false;
+
+ // Avoid too many nested initializations to prevent a stack overflow.
+ if (InitializationChainLength > MaxInitializationChainLength)
+ return false;
+
+ ShouldUpdateAA = shouldUpdateAA<AAType>(IRP);
+
+ return !AAType::hasTrivialInitializer() || ShouldUpdateAA;
+ }
+
+ /// Determine opportunities to derive 'default' attributes in \p F and create
+ /// abstract attribute objects for them.
+ ///
+ /// \param F The function that is checked for attribute opportunities.
+ ///
+ /// Note that abstract attribute instances are generally created even if the
+ /// IR already contains the information they would deduce. The most important
+ /// reason for this is the single interface, the one of the abstract attribute
+ /// instance, which can be queried without the need to look at the IR in
+ /// various places.
+ void identifyDefaultAbstractAttributes(Function &F);
+
+ /// Determine whether the function \p F is IPO amendable
+ ///
+ /// If a function is exactly defined or it has alwaysinline attribute
+ /// and is viable to be inlined, we say it is IPO amendable
+ bool isFunctionIPOAmendable(const Function &F) {
+ return F.hasExactDefinition() || InfoCache.InlineableFunctions.count(&F) ||
+ (Configuration.IPOAmendableCB && Configuration.IPOAmendableCB(F));
+ }
+
+ /// Mark the internal function \p F as live.
+ ///
+ /// This will trigger the identification and initialization of attributes for
+ /// \p F.
+ void markLiveInternalFunction(const Function &F) {
+ assert(F.hasLocalLinkage() &&
+ "Only local linkage is assumed dead initially.");
+
+ if (Configuration.DefaultInitializeLiveInternals)
+ identifyDefaultAbstractAttributes(const_cast<Function &>(F));
+ if (Configuration.InitializationCallback)
+ Configuration.InitializationCallback(*this, F);
+ }
+
+ /// Helper function to remove callsite.
+ void removeCallSite(CallInst *CI) {
+ if (!CI)
+ return;
+
+ Configuration.CGUpdater.removeCallSite(*CI);
+ }
+
+ /// Record that \p U is to be replaces with \p NV after information was
+ /// manifested. This also triggers deletion of trivially dead istructions.
+ bool changeUseAfterManifest(Use &U, Value &NV) {
+ Value *&V = ToBeChangedUses[&U];
+ if (V && (V->stripPointerCasts() == NV.stripPointerCasts() ||
+ isa_and_nonnull<UndefValue>(V)))
+ return false;
+ assert((!V || V == &NV || isa<UndefValue>(NV)) &&
+ "Use was registered twice for replacement with different values!");
+ V = &NV;
+ return true;
+ }
+
+ /// Helper function to replace all uses associated with \p IRP with \p NV.
+ /// Return true if there is any change. The flag \p ChangeDroppable indicates
+ /// if dropppable uses should be changed too.
+ bool changeAfterManifest(const IRPosition IRP, Value &NV,
+ bool ChangeDroppable = true) {
+ if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT) {
+ auto *CB = cast<CallBase>(IRP.getCtxI());
+ return changeUseAfterManifest(
+ CB->getArgOperandUse(IRP.getCallSiteArgNo()), NV);
+ }
+ Value &V = IRP.getAssociatedValue();
+ auto &Entry = ToBeChangedValues[&V];
+ Value *CurNV = get<0>(Entry);
+ if (CurNV && (CurNV->stripPointerCasts() == NV.stripPointerCasts() ||
+ isa<UndefValue>(CurNV)))
+ return false;
+ assert((!CurNV || CurNV == &NV || isa<UndefValue>(NV)) &&
+ "Value replacement was registered twice with different values!");
+ Entry = {&NV, ChangeDroppable};
+ return true;
+ }
+
+ /// Record that \p I is to be replaced with `unreachable` after information
+ /// was manifested.
+ void changeToUnreachableAfterManifest(Instruction *I) {
+ ToBeChangedToUnreachableInsts.insert(I);
+ }
+
+ /// Record that \p II has at least one dead successor block. This information
+ /// is used, e.g., to replace \p II with a call, after information was
+ /// manifested.
+ void registerInvokeWithDeadSuccessor(InvokeInst &II) {
+ InvokeWithDeadSuccessor.insert(&II);
+ }
+
+ /// Record that \p I is deleted after information was manifested. This also
+ /// triggers deletion of trivially dead istructions.
+ void deleteAfterManifest(Instruction &I) { ToBeDeletedInsts.insert(&I); }
+
+ /// Record that \p BB is deleted after information was manifested. This also
+ /// triggers deletion of trivially dead istructions.
+ void deleteAfterManifest(BasicBlock &BB) { ToBeDeletedBlocks.insert(&BB); }
+
+ // Record that \p BB is added during the manifest of an AA. Added basic blocks
+ // are preserved in the IR.
+ void registerManifestAddedBasicBlock(BasicBlock &BB) {
+ ManifestAddedBlocks.insert(&BB);
+ }
+
+ /// Record that \p F is deleted after information was manifested.
+ void deleteAfterManifest(Function &F) {
+ if (Configuration.DeleteFns)
+ ToBeDeletedFunctions.insert(&F);
+ }
+
+ /// Return the attributes of kind \p AK existing in the IR as operand bundles
+ /// of an llvm.assume.
+ bool getAttrsFromAssumes(const IRPosition &IRP, Attribute::AttrKind AK,
+ SmallVectorImpl<Attribute> &Attrs);
+
+ /// Return true if any kind in \p AKs existing in the IR at a position that
+ /// will affect this one. See also getAttrs(...).
+ /// \param IgnoreSubsumingPositions Flag to determine if subsuming positions,
+ /// e.g., the function position if this is an
+ /// argument position, should be ignored.
+ bool hasAttr(const IRPosition &IRP, ArrayRef<Attribute::AttrKind> AKs,
+ bool IgnoreSubsumingPositions = false,
+ Attribute::AttrKind ImpliedAttributeKind = Attribute::None);
+
+ /// Return the attributes of any kind in \p AKs existing in the IR at a
+ /// position that will affect this one. While each position can only have a
+ /// single attribute of any kind in \p AKs, there are "subsuming" positions
+ /// that could have an attribute as well. This method returns all attributes
+ /// found in \p Attrs.
+ /// \param IgnoreSubsumingPositions Flag to determine if subsuming positions,
+ /// e.g., the function position if this is an
+ /// argument position, should be ignored.
+ void getAttrs(const IRPosition &IRP, ArrayRef<Attribute::AttrKind> AKs,
+ SmallVectorImpl<Attribute> &Attrs,
+ bool IgnoreSubsumingPositions = false);
+
+ /// Remove all \p AttrKinds attached to \p IRP.
+ ChangeStatus removeAttrs(const IRPosition &IRP,
+ ArrayRef<Attribute::AttrKind> AttrKinds);
+ ChangeStatus removeAttrs(const IRPosition &IRP, ArrayRef<StringRef> Attrs);
+
+ /// Attach \p DeducedAttrs to \p IRP, if \p ForceReplace is set we do this
+ /// even if the same attribute kind was already present.
+ ChangeStatus manifestAttrs(const IRPosition &IRP,
+ ArrayRef<Attribute> DeducedAttrs,
+ bool ForceReplace = false);
+
+private:
+ /// Helper to check \p Attrs for \p AK, if not found, check if \p
+ /// AAType::isImpliedByIR is true, and if not, create AAType for \p IRP.
+ template <Attribute::AttrKind AK, typename AAType>
+ void checkAndQueryIRAttr(const IRPosition &IRP, AttributeSet Attrs);
+
+ /// Helper to apply \p CB on all attributes of type \p AttrDescs of \p IRP.
+ template <typename DescTy>
+ ChangeStatus updateAttrMap(const IRPosition &IRP, ArrayRef<DescTy> AttrDescs,
+ function_ref<bool(const DescTy &, AttributeSet,
+ AttributeMask &, AttrBuilder &)>
+ CB);
+
+ /// Mapping from functions/call sites to their attributes.
+ DenseMap<Value *, AttributeList> AttrsMap;
+
+public:
+ /// If \p IRP is assumed to be a constant, return it, if it is unclear yet,
+ /// return std::nullopt, otherwise return `nullptr`.
+ std::optional<Constant *> getAssumedConstant(const IRPosition &IRP,
+ const AbstractAttribute &AA,
+ bool &UsedAssumedInformation);
+ std::optional<Constant *> getAssumedConstant(const Value &V,
+ const AbstractAttribute &AA,
+ bool &UsedAssumedInformation) {
+ return getAssumedConstant(IRPosition::value(V), AA, UsedAssumedInformation);
+ }
+
+ /// If \p V is assumed simplified, return it, if it is unclear yet,
+ /// return std::nullopt, otherwise return `nullptr`.
+ std::optional<Value *> getAssumedSimplified(const IRPosition &IRP,
+ const AbstractAttribute &AA,
+ bool &UsedAssumedInformation,
+ AA::ValueScope S) {
+ return getAssumedSimplified(IRP, &AA, UsedAssumedInformation, S);
+ }
+ std::optional<Value *> getAssumedSimplified(const Value &V,
+ const AbstractAttribute &AA,
+ bool &UsedAssumedInformation,
+ AA::ValueScope S) {
+ return getAssumedSimplified(IRPosition::value(V), AA,
+ UsedAssumedInformation, S);
+ }
+
+ /// If \p V is assumed simplified, return it, if it is unclear yet,
+ /// return std::nullopt, otherwise return `nullptr`. Same as the public
+ /// version except that it can be used without recording dependences on any \p
+ /// AA.
+ std::optional<Value *> getAssumedSimplified(const IRPosition &V,
+ const AbstractAttribute *AA,
+ bool &UsedAssumedInformation,
+ AA::ValueScope S);
+
+ /// Try to simplify \p IRP and in the scope \p S. If successful, true is
+ /// returned and all potential values \p IRP can take are put into \p Values.
+ /// If the result in \p Values contains select or PHI instructions it means
+ /// those could not be simplified to a single value. Recursive calls with
+ /// these instructions will yield their respective potential values. If false
+ /// is returned no other information is valid.
+ bool getAssumedSimplifiedValues(const IRPosition &IRP,
+ const AbstractAttribute *AA,
+ SmallVectorImpl<AA::ValueAndContext> &Values,
+ AA::ValueScope S,
+ bool &UsedAssumedInformation,
+ bool RecurseForSelectAndPHI = true);
+
+ /// Register \p CB as a simplification callback.
+ /// `Attributor::getAssumedSimplified` will use these callbacks before
+ /// we it will ask `AAValueSimplify`. It is important to ensure this
+ /// is called before `identifyDefaultAbstractAttributes`, assuming the
+ /// latter is called at all.
+ using SimplifictionCallbackTy = std::function<std::optional<Value *>(
+ const IRPosition &, const AbstractAttribute *, bool &)>;
+ void registerSimplificationCallback(const IRPosition &IRP,
+ const SimplifictionCallbackTy &CB) {
+ SimplificationCallbacks[IRP].emplace_back(CB);
+ }
+
+ /// Return true if there is a simplification callback for \p IRP.
+ bool hasSimplificationCallback(const IRPosition &IRP) {
+ return SimplificationCallbacks.count(IRP);
+ }
+
+ /// Register \p CB as a simplification callback.
+ /// Similar to \p registerSimplificationCallback, the call back will be called
+ /// first when we simplify a global variable \p GV.
+ using GlobalVariableSimplifictionCallbackTy =
+ std::function<std::optional<Constant *>(
+ const GlobalVariable &, const AbstractAttribute *, bool &)>;
+ void registerGlobalVariableSimplificationCallback(
+ const GlobalVariable &GV,
+ const GlobalVariableSimplifictionCallbackTy &CB) {
+ GlobalVariableSimplificationCallbacks[&GV].emplace_back(CB);
+ }
+
+ /// Return true if there is a simplification callback for \p GV.
+ bool hasGlobalVariableSimplificationCallback(const GlobalVariable &GV) {
+ return GlobalVariableSimplificationCallbacks.count(&GV);
+ }
+
+ /// Return \p std::nullopt if there is no call back registered for \p GV or
+ /// the call back is still not sure if \p GV can be simplified. Return \p
+ /// nullptr if \p GV can't be simplified.
+ std::optional<Constant *>
+ getAssumedInitializerFromCallBack(const GlobalVariable &GV,
+ const AbstractAttribute *AA,
+ bool &UsedAssumedInformation) {
+ assert(GlobalVariableSimplificationCallbacks.contains(&GV));
+ for (auto &CB : GlobalVariableSimplificationCallbacks.lookup(&GV)) {
+ auto SimplifiedGV = CB(GV, AA, UsedAssumedInformation);
+ // For now we assume the call back will not return a std::nullopt.
+ assert(SimplifiedGV.has_value() && "SimplifiedGV has not value");
+ return *SimplifiedGV;
+ }
+ llvm_unreachable("there must be a callback registered");
+ }
+
+ using VirtualUseCallbackTy =
+ std::function<bool(Attributor &, const AbstractAttribute *)>;
+ void registerVirtualUseCallback(const Value &V,
+ const VirtualUseCallbackTy &CB) {
+ VirtualUseCallbacks[&V].emplace_back(CB);
+ }
+
+private:
+ /// The vector with all simplification callbacks registered by outside AAs.
+ DenseMap<IRPosition, SmallVector<SimplifictionCallbackTy, 1>>
+ SimplificationCallbacks;
+
+ /// The vector with all simplification callbacks for global variables
+ /// registered by outside AAs.
+ DenseMap<const GlobalVariable *,
+ SmallVector<GlobalVariableSimplifictionCallbackTy, 1>>
+ GlobalVariableSimplificationCallbacks;
+
+ DenseMap<const Value *, SmallVector<VirtualUseCallbackTy, 1>>
+ VirtualUseCallbacks;
+
+public:
+ /// Translate \p V from the callee context into the call site context.
+ std::optional<Value *>
+ translateArgumentToCallSiteContent(std::optional<Value *> V, CallBase &CB,
+ const AbstractAttribute &AA,
+ bool &UsedAssumedInformation);
+
+ /// Return true if \p AA (or its context instruction) is assumed dead.
+ ///
+ /// If \p LivenessAA is not provided it is queried.
+ bool isAssumedDead(const AbstractAttribute &AA, const AAIsDead *LivenessAA,
+ bool &UsedAssumedInformation,
+ bool CheckBBLivenessOnly = false,
+ DepClassTy DepClass = DepClassTy::OPTIONAL);
+
+ /// Return true if \p I is assumed dead.
+ ///
+ /// If \p LivenessAA is not provided it is queried.
+ bool isAssumedDead(const Instruction &I, const AbstractAttribute *QueryingAA,
+ const AAIsDead *LivenessAA, bool &UsedAssumedInformation,
+ bool CheckBBLivenessOnly = false,
+ DepClassTy DepClass = DepClassTy::OPTIONAL,
+ bool CheckForDeadStore = false);
+
+ /// Return true if \p U is assumed dead.
+ ///
+ /// If \p FnLivenessAA is not provided it is queried.
+ bool isAssumedDead(const Use &U, const AbstractAttribute *QueryingAA,
+ const AAIsDead *FnLivenessAA, bool &UsedAssumedInformation,
+ bool CheckBBLivenessOnly = false,
+ DepClassTy DepClass = DepClassTy::OPTIONAL);
+
+ /// Return true if \p IRP is assumed dead.
+ ///
+ /// If \p FnLivenessAA is not provided it is queried.
+ bool isAssumedDead(const IRPosition &IRP, const AbstractAttribute *QueryingAA,
+ const AAIsDead *FnLivenessAA, bool &UsedAssumedInformation,
+ bool CheckBBLivenessOnly = false,
+ DepClassTy DepClass = DepClassTy::OPTIONAL);
+
+ /// Return true if \p BB is assumed dead.
+ ///
+ /// If \p LivenessAA is not provided it is queried.
+ bool isAssumedDead(const BasicBlock &BB, const AbstractAttribute *QueryingAA,
+ const AAIsDead *FnLivenessAA,
+ DepClassTy DepClass = DepClassTy::OPTIONAL);
+
+ /// Check \p Pred on all potential Callees of \p CB.
+ ///
+ /// This method will evaluate \p Pred with all potential callees of \p CB as
+ /// input and return true if \p Pred does. If some callees might be unknown
+ /// this function will return false.
+ bool checkForAllCallees(
+ function_ref<bool(ArrayRef<const Function *> Callees)> Pred,
+ const AbstractAttribute &QueryingAA, const CallBase &CB);
+
+ /// Check \p Pred on all (transitive) uses of \p V.
+ ///
+ /// This method will evaluate \p Pred on all (transitive) uses of the
+ /// associated value and return true if \p Pred holds every time.
+ /// If uses are skipped in favor of equivalent ones, e.g., if we look through
+ /// memory, the \p EquivalentUseCB will be used to give the caller an idea
+ /// what original used was replaced by a new one (or new ones). The visit is
+ /// cut short if \p EquivalentUseCB returns false and the function will return
+ /// false as well.
+ bool checkForAllUses(function_ref<bool(const Use &, bool &)> Pred,
+ const AbstractAttribute &QueryingAA, const Value &V,
+ bool CheckBBLivenessOnly = false,
+ DepClassTy LivenessDepClass = DepClassTy::OPTIONAL,
+ bool IgnoreDroppableUses = true,
+ function_ref<bool(const Use &OldU, const Use &NewU)>
+ EquivalentUseCB = nullptr);
+
+ /// Emit a remark generically.
+ ///
+ /// This template function can be used to generically emit a remark. The
+ /// RemarkKind should be one of the following:
+ /// - OptimizationRemark to indicate a successful optimization attempt
+ /// - OptimizationRemarkMissed to report a failed optimization attempt
+ /// - OptimizationRemarkAnalysis to provide additional information about an
+ /// optimization attempt
+ ///
+ /// The remark is built using a callback function \p RemarkCB that takes a
+ /// RemarkKind as input and returns a RemarkKind.
+ template <typename RemarkKind, typename RemarkCallBack>
+ void emitRemark(Instruction *I, StringRef RemarkName,
+ RemarkCallBack &&RemarkCB) const {
+ if (!Configuration.OREGetter)
+ return;
+
+ Function *F = I->getFunction();
+ auto &ORE = Configuration.OREGetter(F);
+
+ if (RemarkName.starts_with("OMP"))
+ ORE.emit([&]() {
+ return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, I))
+ << " [" << RemarkName << "]";
+ });
+ else
+ ORE.emit([&]() {
+ return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, I));
+ });
+ }
+
+ /// Emit a remark on a function.
+ template <typename RemarkKind, typename RemarkCallBack>
+ void emitRemark(Function *F, StringRef RemarkName,
+ RemarkCallBack &&RemarkCB) const {
+ if (!Configuration.OREGetter)
+ return;
+
+ auto &ORE = Configuration.OREGetter(F);
+
+ if (RemarkName.starts_with("OMP"))
+ ORE.emit([&]() {
+ return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, F))
+ << " [" << RemarkName << "]";
+ });
+ else
+ ORE.emit([&]() {
+ return RemarkCB(RemarkKind(Configuration.PassName, RemarkName, F));
+ });
+ }
+
+ /// Helper struct used in the communication between an abstract attribute (AA)
+ /// that wants to change the signature of a function and the Attributor which
+ /// applies the changes. The struct is partially initialized with the
+ /// information from the AA (see the constructor). All other members are
+ /// provided by the Attributor prior to invoking any callbacks.
+ struct ArgumentReplacementInfo {
+ /// Callee repair callback type
+ ///
+ /// The function repair callback is invoked once to rewire the replacement
+ /// arguments in the body of the new function. The argument replacement info
+ /// is passed, as build from the registerFunctionSignatureRewrite call, as
+ /// well as the replacement function and an iteratore to the first
+ /// replacement argument.
+ using CalleeRepairCBTy = std::function<void(
+ const ArgumentReplacementInfo &, Function &, Function::arg_iterator)>;
+
+ /// Abstract call site (ACS) repair callback type
+ ///
+ /// The abstract call site repair callback is invoked once on every abstract
+ /// call site of the replaced function (\see ReplacedFn). The callback needs
+ /// to provide the operands for the call to the new replacement function.
+ /// The number and type of the operands appended to the provided vector
+ /// (second argument) is defined by the number and types determined through
+ /// the replacement type vector (\see ReplacementTypes). The first argument
+ /// is the ArgumentReplacementInfo object registered with the Attributor
+ /// through the registerFunctionSignatureRewrite call.
+ using ACSRepairCBTy =
+ std::function<void(const ArgumentReplacementInfo &, AbstractCallSite,
+ SmallVectorImpl<Value *> &)>;
+
+ /// Simple getters, see the corresponding members for details.
+ ///{
+
+ Attributor &getAttributor() const { return A; }
+ const Function &getReplacedFn() const { return ReplacedFn; }
+ const Argument &getReplacedArg() const { return ReplacedArg; }
+ unsigned getNumReplacementArgs() const { return ReplacementTypes.size(); }
+ const SmallVectorImpl<Type *> &getReplacementTypes() const {
+ return ReplacementTypes;
+ }
+
+ ///}
+
+ private:
+ /// Constructor that takes the argument to be replaced, the types of
+ /// the replacement arguments, as well as callbacks to repair the call sites
+ /// and new function after the replacement happened.
+ ArgumentReplacementInfo(Attributor &A, Argument &Arg,
+ ArrayRef<Type *> ReplacementTypes,
+ CalleeRepairCBTy &&CalleeRepairCB,
+ ACSRepairCBTy &&ACSRepairCB)
+ : A(A), ReplacedFn(*Arg.getParent()), ReplacedArg(Arg),
+ ReplacementTypes(ReplacementTypes.begin(), ReplacementTypes.end()),
+ CalleeRepairCB(std::move(CalleeRepairCB)),
+ ACSRepairCB(std::move(ACSRepairCB)) {}
+
+ /// Reference to the attributor to allow access from the callbacks.
+ Attributor &A;
+
+ /// The "old" function replaced by ReplacementFn.
+ const Function &ReplacedFn;
+
+ /// The "old" argument replaced by new ones defined via ReplacementTypes.
+ const Argument &ReplacedArg;
+
+ /// The types of the arguments replacing ReplacedArg.
+ const SmallVector<Type *, 8> ReplacementTypes;
+
+ /// Callee repair callback, see CalleeRepairCBTy.
+ const CalleeRepairCBTy CalleeRepairCB;
+
+ /// Abstract call site (ACS) repair callback, see ACSRepairCBTy.
+ const ACSRepairCBTy ACSRepairCB;
+
+ /// Allow access to the private members from the Attributor.
+ friend struct Attributor;
+ };
+
+ /// Check if we can rewrite a function signature.
+ ///
+ /// The argument \p Arg is replaced with new ones defined by the number,
+ /// order, and types in \p ReplacementTypes.
+ ///
+ /// \returns True, if the replacement can be registered, via
+ /// registerFunctionSignatureRewrite, false otherwise.
+ bool isValidFunctionSignatureRewrite(Argument &Arg,
+ ArrayRef<Type *> ReplacementTypes);
+
+ /// Register a rewrite for a function signature.
+ ///
+ /// The argument \p Arg is replaced with new ones defined by the number,
+ /// order, and types in \p ReplacementTypes. The rewiring at the call sites is
+ /// done through \p ACSRepairCB and at the callee site through
+ /// \p CalleeRepairCB.
+ ///
+ /// \returns True, if the replacement was registered, false otherwise.
+ bool registerFunctionSignatureRewrite(
+ Argument &Arg, ArrayRef<Type *> ReplacementTypes,
+ ArgumentReplacementInfo::CalleeRepairCBTy &&CalleeRepairCB,
+ ArgumentReplacementInfo::ACSRepairCBTy &&ACSRepairCB);
+
+ /// Check \p Pred on all function call sites.
+ ///
+ /// This method will evaluate \p Pred on call sites and return
+ /// true if \p Pred holds in every call sites. However, this is only possible
+ /// all call sites are known, hence the function has internal linkage.
+ /// If true is returned, \p UsedAssumedInformation is set if assumed
+ /// information was used to skip or simplify potential call sites.
+ bool checkForAllCallSites(function_ref<bool(AbstractCallSite)> Pred,
+ const AbstractAttribute &QueryingAA,
+ bool RequireAllCallSites,
+ bool &UsedAssumedInformation);
+
+ /// Check \p Pred on all call sites of \p Fn.
+ ///
+ /// This method will evaluate \p Pred on call sites and return
+ /// true if \p Pred holds in every call sites. However, this is only possible
+ /// all call sites are known, hence the function has internal linkage.
+ /// If true is returned, \p UsedAssumedInformation is set if assumed
+ /// information was used to skip or simplify potential call sites.
+ bool checkForAllCallSites(function_ref<bool(AbstractCallSite)> Pred,
+ const Function &Fn, bool RequireAllCallSites,
+ const AbstractAttribute *QueryingAA,
+ bool &UsedAssumedInformation,
+ bool CheckPotentiallyDead = false);
+
+ /// Check \p Pred on all values potentially returned by the function
+ /// associated with \p QueryingAA.
+ ///
+ /// This is the context insensitive version of the method above.
+ bool
+ checkForAllReturnedValues(function_ref<bool(Value &)> Pred,
+ const AbstractAttribute &QueryingAA,
+ AA::ValueScope S = AA::ValueScope::Intraprocedural,
+ bool RecurseForSelectAndPHI = true);
+
+ /// Check \p Pred on all instructions in \p Fn with an opcode present in
+ /// \p Opcodes.
+ ///
+ /// This method will evaluate \p Pred on all instructions with an opcode
+ /// present in \p Opcode and return true if \p Pred holds on all of them.
+ bool checkForAllInstructions(function_ref<bool(Instruction &)> Pred,
+ const Function *Fn,
+ const AbstractAttribute *QueryingAA,
+ ArrayRef<unsigned> Opcodes,
+ bool &UsedAssumedInformation,
+ bool CheckBBLivenessOnly = false,
+ bool CheckPotentiallyDead = false);
+
+ /// Check \p Pred on all instructions with an opcode present in \p Opcodes.
+ ///
+ /// This method will evaluate \p Pred on all instructions with an opcode
+ /// present in \p Opcode and return true if \p Pred holds on all of them.
+ bool checkForAllInstructions(function_ref<bool(Instruction &)> Pred,
+ const AbstractAttribute &QueryingAA,
+ ArrayRef<unsigned> Opcodes,
+ bool &UsedAssumedInformation,
+ bool CheckBBLivenessOnly = false,
+ bool CheckPotentiallyDead = false);
+
+ /// Check \p Pred on all call-like instructions (=CallBased derived).
+ ///
+ /// See checkForAllCallLikeInstructions(...) for more information.
+ bool checkForAllCallLikeInstructions(function_ref<bool(Instruction &)> Pred,
+ const AbstractAttribute &QueryingAA,
+ bool &UsedAssumedInformation,
+ bool CheckBBLivenessOnly = false,
+ bool CheckPotentiallyDead = false) {
+ return checkForAllInstructions(
+ Pred, QueryingAA,
+ {(unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
+ (unsigned)Instruction::Call},
+ UsedAssumedInformation, CheckBBLivenessOnly, CheckPotentiallyDead);
+ }
+
+ /// Check \p Pred on all Read/Write instructions.
+ ///
+ /// This method will evaluate \p Pred on all instructions that read or write
+ /// to memory present in the information cache and return true if \p Pred
+ /// holds on all of them.
+ bool checkForAllReadWriteInstructions(function_ref<bool(Instruction &)> Pred,
+ AbstractAttribute &QueryingAA,
+ bool &UsedAssumedInformation);
+
+ /// Create a shallow wrapper for \p F such that \p F has internal linkage
+ /// afterwards. It also sets the original \p F 's name to anonymous
+ ///
+ /// A wrapper is a function with the same type (and attributes) as \p F
+ /// that will only call \p F and return the result, if any.
+ ///
+ /// Assuming the declaration of looks like:
+ /// rty F(aty0 arg0, ..., atyN argN);
+ ///
+ /// The wrapper will then look as follows:
+ /// rty wrapper(aty0 arg0, ..., atyN argN) {
+ /// return F(arg0, ..., argN);
+ /// }
+ ///
+ static void createShallowWrapper(Function &F);
+
+ /// Returns true if the function \p F can be internalized. i.e. it has a
+ /// compatible linkage.
+ static bool isInternalizable(Function &F);
+
+ /// Make another copy of the function \p F such that the copied version has
+ /// internal linkage afterwards and can be analysed. Then we replace all uses
+ /// of the original function to the copied one
+ ///
+ /// Only non-locally linked functions that have `linkonce_odr` or `weak_odr`
+ /// linkage can be internalized because these linkages guarantee that other
+ /// definitions with the same name have the same semantics as this one.
+ ///
+ /// This will only be run if the `attributor-allow-deep-wrappers` option is
+ /// set, or if the function is called with \p Force set to true.
+ ///
+ /// If the function \p F failed to be internalized the return value will be a
+ /// null pointer.
+ static Function *internalizeFunction(Function &F, bool Force = false);
+
+ /// Make copies of each function in the set \p FnSet such that the copied
+ /// version has internal linkage afterwards and can be analysed. Then we
+ /// replace all uses of the original function to the copied one. The map
+ /// \p FnMap contains a mapping of functions to their internalized versions.
+ ///
+ /// Only non-locally linked functions that have `linkonce_odr` or `weak_odr`
+ /// linkage can be internalized because these linkages guarantee that other
+ /// definitions with the same name have the same semantics as this one.
+ ///
+ /// This version will internalize all the functions in the set \p FnSet at
+ /// once and then replace the uses. This prevents internalized functions being
+ /// called by external functions when there is an internalized version in the
+ /// module.
+ static bool internalizeFunctions(SmallPtrSetImpl<Function *> &FnSet,
+ DenseMap<Function *, Function *> &FnMap);
+
+ /// Return the data layout associated with the anchor scope.
+ const DataLayout &getDataLayout() const { return InfoCache.DL; }
+
+ /// The allocator used to allocate memory, e.g. for `AbstractAttribute`s.
+ BumpPtrAllocator &Allocator;
+
+ const SmallSetVector<Function *, 8> &getModifiedFunctions() {
+ return CGModifiedFunctions;
+ }
+
+private:
+ /// This method will do fixpoint iteration until fixpoint or the
+ /// maximum iteration count is reached.
+ ///
+ /// If the maximum iteration count is reached, This method will
+ /// indicate pessimistic fixpoint on attributes that transitively depend
+ /// on attributes that were scheduled for an update.
+ void runTillFixpoint();
+
+ /// Gets called after scheduling, manifests attributes to the LLVM IR.
+ ChangeStatus manifestAttributes();
+
+ /// Gets called after attributes have been manifested, cleans up the IR.
+ /// Deletes dead functions, blocks and instructions.
+ /// Rewrites function signitures and updates the call graph.
+ ChangeStatus cleanupIR();
+
+ /// Identify internal functions that are effectively dead, thus not reachable
+ /// from a live entry point. The functions are added to ToBeDeletedFunctions.
+ void identifyDeadInternalFunctions();
+
+ /// Run `::update` on \p AA and track the dependences queried while doing so.
+ /// Also adjust the state if we know further updates are not necessary.
+ ChangeStatus updateAA(AbstractAttribute &AA);
+
+ /// Remember the dependences on the top of the dependence stack such that they
+ /// may trigger further updates. (\see DependenceStack)
+ void rememberDependences();
+
+ /// Determine if CallBase context in \p IRP should be propagated.
+ bool shouldPropagateCallBaseContext(const IRPosition &IRP);
+
+ /// Apply all requested function signature rewrites
+ /// (\see registerFunctionSignatureRewrite) and return Changed if the module
+ /// was altered.
+ ChangeStatus
+ rewriteFunctionSignatures(SmallSetVector<Function *, 8> &ModifiedFns);
+
+ /// Check if the Attribute \p AA should be seeded.
+ /// See getOrCreateAAFor.
+ bool shouldSeedAttribute(AbstractAttribute &AA);
+
+ /// A nested map to lookup abstract attributes based on the argument position
+ /// on the outer level, and the addresses of the static member (AAType::ID) on
+ /// the inner level.
+ ///{
+ using AAMapKeyTy = std::pair<const char *, IRPosition>;
+ DenseMap<AAMapKeyTy, AbstractAttribute *> AAMap;
+ ///}
+
+ /// Map to remember all requested signature changes (= argument replacements).
+ DenseMap<Function *, SmallVector<std::unique_ptr<ArgumentReplacementInfo>, 8>>
+ ArgumentReplacementMap;
+
+ /// The set of functions we are deriving attributes for.
+ SetVector<Function *> &Functions;
+
+ /// The information cache that holds pre-processed (LLVM-IR) information.
+ InformationCache &InfoCache;
+
+ /// Abstract Attribute dependency graph
+ AADepGraph DG;
+
+ /// Set of functions for which we modified the content such that it might
+ /// impact the call graph.
+ SmallSetVector<Function *, 8> CGModifiedFunctions;
+
+ /// Information about a dependence. If FromAA is changed ToAA needs to be
+ /// updated as well.
+ struct DepInfo {
+ const AbstractAttribute *FromAA;
+ const AbstractAttribute *ToAA;
+ DepClassTy DepClass;
+ };
+
+ /// The dependence stack is used to track dependences during an
+ /// `AbstractAttribute::update` call. As `AbstractAttribute::update` can be
+ /// recursive we might have multiple vectors of dependences in here. The stack
+ /// size, should be adjusted according to the expected recursion depth and the
+ /// inner dependence vector size to the expected number of dependences per
+ /// abstract attribute. Since the inner vectors are actually allocated on the
+ /// stack we can be generous with their size.
+ using DependenceVector = SmallVector<DepInfo, 8>;
+ SmallVector<DependenceVector *, 16> DependenceStack;
+
+ /// A set to remember the functions we already assume to be live and visited.
+ DenseSet<const Function *> VisitedFunctions;
+
+ /// Uses we replace with a new value after manifest is done. We will remove
+ /// then trivially dead instructions as well.
+ SmallMapVector<Use *, Value *, 32> ToBeChangedUses;
+
+ /// Values we replace with a new value after manifest is done. We will remove
+ /// then trivially dead instructions as well.
+ SmallMapVector<Value *, PointerIntPair<Value *, 1, bool>, 32>
+ ToBeChangedValues;
+
+ /// Instructions we replace with `unreachable` insts after manifest is done.
+ SmallSetVector<WeakVH, 16> ToBeChangedToUnreachableInsts;
+
+ /// Invoke instructions with at least a single dead successor block.
+ SmallSetVector<WeakVH, 16> InvokeWithDeadSuccessor;
+
+ /// A flag that indicates which stage of the process we are in. Initially, the
+ /// phase is SEEDING. Phase is changed in `Attributor::run()`
+ enum class AttributorPhase {
+ SEEDING,
+ UPDATE,
+ MANIFEST,
+ CLEANUP,
+ } Phase = AttributorPhase::SEEDING;
+
+ /// The current initialization chain length. Tracked to avoid stack overflows.
+ unsigned InitializationChainLength = 0;
+
+ /// Functions, blocks, and instructions we delete after manifest is done.
+ ///
+ ///{
+ SmallPtrSet<BasicBlock *, 8> ManifestAddedBlocks;
+ SmallSetVector<Function *, 8> ToBeDeletedFunctions;
+ SmallSetVector<BasicBlock *, 8> ToBeDeletedBlocks;
+ SmallSetVector<WeakVH, 8> ToBeDeletedInsts;
+ ///}
+
+ /// Container with all the query AAs that requested an update via
+ /// registerForUpdate.
+ SmallSetVector<AbstractAttribute *, 16> QueryAAsAwaitingUpdate;
+
+ /// User provided configuration for this Attributor instance.
+ const AttributorConfig Configuration;
+
+ friend AADepGraph;
+ friend AttributorCallGraph;
+};
+
+/// An interface to query the internal state of an abstract attribute.
+///
+/// The abstract state is a minimal interface that allows the Attributor to
+/// communicate with the abstract attributes about their internal state without
+/// enforcing or exposing implementation details, e.g., the (existence of an)
+/// underlying lattice.
+///
+/// It is sufficient to be able to query if a state is (1) valid or invalid, (2)
+/// at a fixpoint, and to indicate to the state that (3) an optimistic fixpoint
+/// was reached or (4) a pessimistic fixpoint was enforced.
+///
+/// All methods need to be implemented by the subclass. For the common use case,
+/// a single boolean state or a bit-encoded state, the BooleanState and
+/// {Inc,Dec,Bit}IntegerState classes are already provided. An abstract
+/// attribute can inherit from them to get the abstract state interface and
+/// additional methods to directly modify the state based if needed. See the
+/// class comments for help.
+struct AbstractState {
+ virtual ~AbstractState() = default;
+
+ /// Return if this abstract state is in a valid state. If false, no
+ /// information provided should be used.
+ virtual bool isValidState() const = 0;
+
+ /// Return if this abstract state is fixed, thus does not need to be updated
+ /// if information changes as it cannot change itself.
+ virtual bool isAtFixpoint() const = 0;
+
+ /// Indicate that the abstract state should converge to the optimistic state.
+ ///
+ /// This will usually make the optimistically assumed state the known to be
+ /// true state.
+ ///
+ /// \returns ChangeStatus::UNCHANGED as the assumed value should not change.
+ virtual ChangeStatus indicateOptimisticFixpoint() = 0;
+
+ /// Indicate that the abstract state should converge to the pessimistic state.
+ ///
+ /// This will usually revert the optimistically assumed state to the known to
+ /// be true state.
+ ///
+ /// \returns ChangeStatus::CHANGED as the assumed value may change.
+ virtual ChangeStatus indicatePessimisticFixpoint() = 0;
+};
+
+/// Simple state with integers encoding.
+///
+/// The interface ensures that the assumed bits are always a subset of the known
+/// bits. Users can only add known bits and, except through adding known bits,
+/// they can only remove assumed bits. This should guarantee monotonicity and
+/// thereby the existence of a fixpoint (if used correctly). The fixpoint is
+/// reached when the assumed and known state/bits are equal. Users can
+/// force/inidicate a fixpoint. If an optimistic one is indicated, the known
+/// state will catch up with the assumed one, for a pessimistic fixpoint it is
+/// the other way around.
+template <typename base_ty, base_ty BestState, base_ty WorstState>
+struct IntegerStateBase : public AbstractState {
+ using base_t = base_ty;
+
+ IntegerStateBase() = default;
+ IntegerStateBase(base_t Assumed) : Assumed(Assumed) {}
+
+ /// Return the best possible representable state.
+ static constexpr base_t getBestState() { return BestState; }
+ static constexpr base_t getBestState(const IntegerStateBase &) {
+ return getBestState();
+ }
+
+ /// Return the worst possible representable state.
+ static constexpr base_t getWorstState() { return WorstState; }
+ static constexpr base_t getWorstState(const IntegerStateBase &) {
+ return getWorstState();
+ }
+
+ /// See AbstractState::isValidState()
+ /// NOTE: For now we simply pretend that the worst possible state is invalid.
+ bool isValidState() const override { return Assumed != getWorstState(); }
+
+ /// See AbstractState::isAtFixpoint()
+ bool isAtFixpoint() const override { return Assumed == Known; }
+
+ /// See AbstractState::indicateOptimisticFixpoint(...)
+ ChangeStatus indicateOptimisticFixpoint() override {
+ Known = Assumed;
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractState::indicatePessimisticFixpoint(...)
+ ChangeStatus indicatePessimisticFixpoint() override {
+ Assumed = Known;
+ return ChangeStatus::CHANGED;
+ }
+
+ /// Return the known state encoding
+ base_t getKnown() const { return Known; }
+
+ /// Return the assumed state encoding.
+ base_t getAssumed() const { return Assumed; }
+
+ /// Equality for IntegerStateBase.
+ bool
+ operator==(const IntegerStateBase<base_t, BestState, WorstState> &R) const {
+ return this->getAssumed() == R.getAssumed() &&
+ this->getKnown() == R.getKnown();
+ }
+
+ /// Inequality for IntegerStateBase.
+ bool
+ operator!=(const IntegerStateBase<base_t, BestState, WorstState> &R) const {
+ return !(*this == R);
+ }
+
+ /// "Clamp" this state with \p R. The result is subtype dependent but it is
+ /// intended that only information assumed in both states will be assumed in
+ /// this one afterwards.
+ void operator^=(const IntegerStateBase<base_t, BestState, WorstState> &R) {
+ handleNewAssumedValue(R.getAssumed());
+ }
+
+ /// "Clamp" this state with \p R. The result is subtype dependent but it is
+ /// intended that information known in either state will be known in
+ /// this one afterwards.
+ void operator+=(const IntegerStateBase<base_t, BestState, WorstState> &R) {
+ handleNewKnownValue(R.getKnown());
+ }
+
+ void operator|=(const IntegerStateBase<base_t, BestState, WorstState> &R) {
+ joinOR(R.getAssumed(), R.getKnown());
+ }
+
+ void operator&=(const IntegerStateBase<base_t, BestState, WorstState> &R) {
+ joinAND(R.getAssumed(), R.getKnown());
+ }
+
+protected:
+ /// Handle a new assumed value \p Value. Subtype dependent.
+ virtual void handleNewAssumedValue(base_t Value) = 0;
+
+ /// Handle a new known value \p Value. Subtype dependent.
+ virtual void handleNewKnownValue(base_t Value) = 0;
+
+ /// Handle a value \p Value. Subtype dependent.
+ virtual void joinOR(base_t AssumedValue, base_t KnownValue) = 0;
+
+ /// Handle a new assumed value \p Value. Subtype dependent.
+ virtual void joinAND(base_t AssumedValue, base_t KnownValue) = 0;
+
+ /// The known state encoding in an integer of type base_t.
+ base_t Known = getWorstState();
+
+ /// The assumed state encoding in an integer of type base_t.
+ base_t Assumed = getBestState();
+};
+
+/// Specialization of the integer state for a bit-wise encoding.
+template <typename base_ty = uint32_t, base_ty BestState = ~base_ty(0),
+ base_ty WorstState = 0>
+struct BitIntegerState
+ : public IntegerStateBase<base_ty, BestState, WorstState> {
+ using super = IntegerStateBase<base_ty, BestState, WorstState>;
+ using base_t = base_ty;
+ BitIntegerState() = default;
+ BitIntegerState(base_t Assumed) : super(Assumed) {}
+
+ /// Return true if the bits set in \p BitsEncoding are "known bits".
+ bool isKnown(base_t BitsEncoding = BestState) const {
+ return (this->Known & BitsEncoding) == BitsEncoding;
+ }
+
+ /// Return true if the bits set in \p BitsEncoding are "assumed bits".
+ bool isAssumed(base_t BitsEncoding = BestState) const {
+ return (this->Assumed & BitsEncoding) == BitsEncoding;
+ }
+
+ /// Add the bits in \p BitsEncoding to the "known bits".
+ BitIntegerState &addKnownBits(base_t Bits) {
+ // Make sure we never miss any "known bits".
+ this->Assumed |= Bits;
+ this->Known |= Bits;
+ return *this;
+ }
+
+ /// Remove the bits in \p BitsEncoding from the "assumed bits" if not known.
+ BitIntegerState &removeAssumedBits(base_t BitsEncoding) {
+ return intersectAssumedBits(~BitsEncoding);
+ }
+
+ /// Remove the bits in \p BitsEncoding from the "known bits".
+ BitIntegerState &removeKnownBits(base_t BitsEncoding) {
+ this->Known = (this->Known & ~BitsEncoding);
+ return *this;
+ }
+
+ /// Keep only "assumed bits" also set in \p BitsEncoding but all known ones.
+ BitIntegerState &intersectAssumedBits(base_t BitsEncoding) {
+ // Make sure we never lose any "known bits".
+ this->Assumed = (this->Assumed & BitsEncoding) | this->Known;
+ return *this;
+ }
+
+private:
+ void handleNewAssumedValue(base_t Value) override {
+ intersectAssumedBits(Value);
+ }
+ void handleNewKnownValue(base_t Value) override { addKnownBits(Value); }
+ void joinOR(base_t AssumedValue, base_t KnownValue) override {
+ this->Known |= KnownValue;
+ this->Assumed |= AssumedValue;
+ }
+ void joinAND(base_t AssumedValue, base_t KnownValue) override {
+ this->Known &= KnownValue;
+ this->Assumed &= AssumedValue;
+ }
+};
+
+/// Specialization of the integer state for an increasing value, hence ~0u is
+/// the best state and 0 the worst.
+template <typename base_ty = uint32_t, base_ty BestState = ~base_ty(0),
+ base_ty WorstState = 0>
+struct IncIntegerState
+ : public IntegerStateBase<base_ty, BestState, WorstState> {
+ using super = IntegerStateBase<base_ty, BestState, WorstState>;
+ using base_t = base_ty;
+
+ IncIntegerState() : super() {}
+ IncIntegerState(base_t Assumed) : super(Assumed) {}
+
+ /// Return the best possible representable state.
+ static constexpr base_t getBestState() { return BestState; }
+ static constexpr base_t
+ getBestState(const IncIntegerState<base_ty, BestState, WorstState> &) {
+ return getBestState();
+ }
+
+ /// Take minimum of assumed and \p Value.
+ IncIntegerState &takeAssumedMinimum(base_t Value) {
+ // Make sure we never lose "known value".
+ this->Assumed = std::max(std::min(this->Assumed, Value), this->Known);
+ return *this;
+ }
+
+ /// Take maximum of known and \p Value.
+ IncIntegerState &takeKnownMaximum(base_t Value) {
+ // Make sure we never lose "known value".
+ this->Assumed = std::max(Value, this->Assumed);
+ this->Known = std::max(Value, this->Known);
+ return *this;
+ }
+
+private:
+ void handleNewAssumedValue(base_t Value) override {
+ takeAssumedMinimum(Value);
+ }
+ void handleNewKnownValue(base_t Value) override { takeKnownMaximum(Value); }
+ void joinOR(base_t AssumedValue, base_t KnownValue) override {
+ this->Known = std::max(this->Known, KnownValue);
+ this->Assumed = std::max(this->Assumed, AssumedValue);
+ }
+ void joinAND(base_t AssumedValue, base_t KnownValue) override {
+ this->Known = std::min(this->Known, KnownValue);
+ this->Assumed = std::min(this->Assumed, AssumedValue);
+ }
+};
+
+/// Specialization of the integer state for a decreasing value, hence 0 is the
+/// best state and ~0u the worst.
+template <typename base_ty = uint32_t>
+struct DecIntegerState : public IntegerStateBase<base_ty, 0, ~base_ty(0)> {
+ using base_t = base_ty;
+
+ /// Take maximum of assumed and \p Value.
+ DecIntegerState &takeAssumedMaximum(base_t Value) {
+ // Make sure we never lose "known value".
+ this->Assumed = std::min(std::max(this->Assumed, Value), this->Known);
+ return *this;
+ }
+
+ /// Take minimum of known and \p Value.
+ DecIntegerState &takeKnownMinimum(base_t Value) {
+ // Make sure we never lose "known value".
+ this->Assumed = std::min(Value, this->Assumed);
+ this->Known = std::min(Value, this->Known);
+ return *this;
+ }
+
+private:
+ void handleNewAssumedValue(base_t Value) override {
+ takeAssumedMaximum(Value);
+ }
+ void handleNewKnownValue(base_t Value) override { takeKnownMinimum(Value); }
+ void joinOR(base_t AssumedValue, base_t KnownValue) override {
+ this->Assumed = std::min(this->Assumed, KnownValue);
+ this->Assumed = std::min(this->Assumed, AssumedValue);
+ }
+ void joinAND(base_t AssumedValue, base_t KnownValue) override {
+ this->Assumed = std::max(this->Assumed, KnownValue);
+ this->Assumed = std::max(this->Assumed, AssumedValue);
+ }
+};
+
+/// Simple wrapper for a single bit (boolean) state.
+struct BooleanState : public IntegerStateBase<bool, true, false> {
+ using super = IntegerStateBase<bool, true, false>;
+ using base_t = IntegerStateBase::base_t;
+
+ BooleanState() = default;
+ BooleanState(base_t Assumed) : super(Assumed) {}
+
+ /// Set the assumed value to \p Value but never below the known one.
+ void setAssumed(bool Value) { Assumed &= (Known | Value); }
+
+ /// Set the known and asssumed value to \p Value.
+ void setKnown(bool Value) {
+ Known |= Value;
+ Assumed |= Value;
+ }
+
+ /// Return true if the state is assumed to hold.
+ bool isAssumed() const { return getAssumed(); }
+
+ /// Return true if the state is known to hold.
+ bool isKnown() const { return getKnown(); }
+
+private:
+ void handleNewAssumedValue(base_t Value) override {
+ if (!Value)
+ Assumed = Known;
+ }
+ void handleNewKnownValue(base_t Value) override {
+ if (Value)
+ Known = (Assumed = Value);
+ }
+ void joinOR(base_t AssumedValue, base_t KnownValue) override {
+ Known |= KnownValue;
+ Assumed |= AssumedValue;
+ }
+ void joinAND(base_t AssumedValue, base_t KnownValue) override {
+ Known &= KnownValue;
+ Assumed &= AssumedValue;
+ }
+};
+
+/// State for an integer range.
+struct IntegerRangeState : public AbstractState {
+
+ /// Bitwidth of the associated value.
+ uint32_t BitWidth;
+
+ /// State representing assumed range, initially set to empty.
+ ConstantRange Assumed;
+
+ /// State representing known range, initially set to [-inf, inf].
+ ConstantRange Known;
+
+ IntegerRangeState(uint32_t BitWidth)
+ : BitWidth(BitWidth), Assumed(ConstantRange::getEmpty(BitWidth)),
+ Known(ConstantRange::getFull(BitWidth)) {}
+
+ IntegerRangeState(const ConstantRange &CR)
+ : BitWidth(CR.getBitWidth()), Assumed(CR),
+ Known(getWorstState(CR.getBitWidth())) {}
+
+ /// Return the worst possible representable state.
+ static ConstantRange getWorstState(uint32_t BitWidth) {
+ return ConstantRange::getFull(BitWidth);
+ }
+
+ /// Return the best possible representable state.
+ static ConstantRange getBestState(uint32_t BitWidth) {
+ return ConstantRange::getEmpty(BitWidth);
+ }
+ static ConstantRange getBestState(const IntegerRangeState &IRS) {
+ return getBestState(IRS.getBitWidth());
+ }
+
+ /// Return associated values' bit width.
+ uint32_t getBitWidth() const { return BitWidth; }
+
+ /// See AbstractState::isValidState()
+ bool isValidState() const override {
+ return BitWidth > 0 && !Assumed.isFullSet();
+ }
+
+ /// See AbstractState::isAtFixpoint()
+ bool isAtFixpoint() const override { return Assumed == Known; }
+
+ /// See AbstractState::indicateOptimisticFixpoint(...)
+ ChangeStatus indicateOptimisticFixpoint() override {
+ Known = Assumed;
+ return ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractState::indicatePessimisticFixpoint(...)
+ ChangeStatus indicatePessimisticFixpoint() override {
+ Assumed = Known;
+ return ChangeStatus::CHANGED;
+ }
+
+ /// Return the known state encoding
+ ConstantRange getKnown() const { return Known; }
+
+ /// Return the assumed state encoding.
+ ConstantRange getAssumed() const { return Assumed; }
+
+ /// Unite assumed range with the passed state.
+ void unionAssumed(const ConstantRange &R) {
+ // Don't lose a known range.
+ Assumed = Assumed.unionWith(R).intersectWith(Known);
+ }
+
+ /// See IntegerRangeState::unionAssumed(..).
+ void unionAssumed(const IntegerRangeState &R) {
+ unionAssumed(R.getAssumed());
+ }
+
+ /// Intersect known range with the passed state.
+ void intersectKnown(const ConstantRange &R) {
+ Assumed = Assumed.intersectWith(R);
+ Known = Known.intersectWith(R);
+ }
+
+ /// See IntegerRangeState::intersectKnown(..).
+ void intersectKnown(const IntegerRangeState &R) {
+ intersectKnown(R.getKnown());
+ }
+
+ /// Equality for IntegerRangeState.
+ bool operator==(const IntegerRangeState &R) const {
+ return getAssumed() == R.getAssumed() && getKnown() == R.getKnown();
+ }
+
+ /// "Clamp" this state with \p R. The result is subtype dependent but it is
+ /// intended that only information assumed in both states will be assumed in
+ /// this one afterwards.
+ IntegerRangeState operator^=(const IntegerRangeState &R) {
+ // NOTE: `^=` operator seems like `intersect` but in this case, we need to
+ // take `union`.
+ unionAssumed(R);
+ return *this;
+ }
+
+ IntegerRangeState operator&=(const IntegerRangeState &R) {
+ // NOTE: `&=` operator seems like `intersect` but in this case, we need to
+ // take `union`.
+ Known = Known.unionWith(R.getKnown());
+ Assumed = Assumed.unionWith(R.getAssumed());
+ return *this;
+ }
+};
+
+/// Simple state for a set.
+///
+/// This represents a state containing a set of values. The interface supports
+/// modelling sets that contain all possible elements. The state's internal
+/// value is modified using union or intersection operations.
+template <typename BaseTy> struct SetState : public AbstractState {
+ /// A wrapper around a set that has semantics for handling unions and
+ /// intersections with a "universal" set that contains all elements.
+ struct SetContents {
+ /// Creates a universal set with no concrete elements or an empty set.
+ SetContents(bool Universal) : Universal(Universal) {}
+
+ /// Creates a non-universal set with concrete values.
+ SetContents(const DenseSet<BaseTy> &Assumptions)
+ : Universal(false), Set(Assumptions) {}
+
+ SetContents(bool Universal, const DenseSet<BaseTy> &Assumptions)
+ : Universal(Universal), Set(Assumptions) {}
+
+ const DenseSet<BaseTy> &getSet() const { return Set; }
+
+ bool isUniversal() const { return Universal; }
+
+ bool empty() const { return Set.empty() && !Universal; }
+
+ /// Finds A := A ^ B where A or B could be the "Universal" set which
+ /// contains every possible attribute. Returns true if changes were made.
+ bool getIntersection(const SetContents &RHS) {
+ bool IsUniversal = Universal;
+ unsigned Size = Set.size();
+
+ // A := A ^ U = A
+ if (RHS.isUniversal())
+ return false;
+
+ // A := U ^ B = B
+ if (Universal)
+ Set = RHS.getSet();
+ else
+ set_intersect(Set, RHS.getSet());
+
+ Universal &= RHS.isUniversal();
+ return IsUniversal != Universal || Size != Set.size();
+ }
+
+ /// Finds A := A u B where A or B could be the "Universal" set which
+ /// contains every possible attribute. returns true if changes were made.
+ bool getUnion(const SetContents &RHS) {
+ bool IsUniversal = Universal;
+ unsigned Size = Set.size();
+
+ // A := A u U = U = U u B
+ if (!RHS.isUniversal() && !Universal)
+ set_union(Set, RHS.getSet());
+
+ Universal |= RHS.isUniversal();
+ return IsUniversal != Universal || Size != Set.size();
+ }
+
+ private:
+ /// Indicates if this set is "universal", containing every possible element.
+ bool Universal;
+
+ /// The set of currently active assumptions.
+ DenseSet<BaseTy> Set;
+ };
+
+ SetState() : Known(false), Assumed(true), IsAtFixedpoint(false) {}
+
+ /// Initializes the known state with an initial set and initializes the
+ /// assumed state as universal.
+ SetState(const DenseSet<BaseTy> &Known)
+ : Known(Known), Assumed(true), IsAtFixedpoint(false) {}
+
+ /// See AbstractState::isValidState()
+ bool isValidState() const override { return !Assumed.empty(); }
+
+ /// See AbstractState::isAtFixpoint()
+ bool isAtFixpoint() const override { return IsAtFixedpoint; }
+
+ /// See AbstractState::indicateOptimisticFixpoint(...)
+ ChangeStatus indicateOptimisticFixpoint() override {
+ IsAtFixedpoint = true;
+ Known = Assumed;
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractState::indicatePessimisticFixpoint(...)
+ ChangeStatus indicatePessimisticFixpoint() override {
+ IsAtFixedpoint = true;
+ Assumed = Known;
+ return ChangeStatus::CHANGED;
+ }
+
+ /// Return the known state encoding.
+ const SetContents &getKnown() const { return Known; }
+
+ /// Return the assumed state encoding.
+ const SetContents &getAssumed() const { return Assumed; }
+
+ /// Returns if the set state contains the element.
+ bool setContains(const BaseTy &Elem) const {
+ return Assumed.getSet().contains(Elem) || Known.getSet().contains(Elem);
+ }
+
+ /// Performs the set intersection between this set and \p RHS. Returns true if
+ /// changes were made.
+ bool getIntersection(const SetContents &RHS) {
+ bool IsUniversal = Assumed.isUniversal();
+ unsigned SizeBefore = Assumed.getSet().size();
+
+ // Get intersection and make sure that the known set is still a proper
+ // subset of the assumed set. A := K u (A ^ R).
+ Assumed.getIntersection(RHS);
+ Assumed.getUnion(Known);
+
+ return SizeBefore != Assumed.getSet().size() ||
+ IsUniversal != Assumed.isUniversal();
+ }
+
+ /// Performs the set union between this set and \p RHS. Returns true if
+ /// changes were made.
+ bool getUnion(const SetContents &RHS) { return Assumed.getUnion(RHS); }
+
+private:
+ /// The set of values known for this state.
+ SetContents Known;
+
+ /// The set of assumed values for this state.
+ SetContents Assumed;
+
+ bool IsAtFixedpoint;
+};
+
+/// Helper to tie a abstract state implementation to an abstract attribute.
+template <typename StateTy, typename BaseType, class... Ts>
+struct StateWrapper : public BaseType, public StateTy {
+ /// Provide static access to the type of the state.
+ using StateType = StateTy;
+
+ StateWrapper(const IRPosition &IRP, Ts... Args)
+ : BaseType(IRP), StateTy(Args...) {}
+
+ /// See AbstractAttribute::getState(...).
+ StateType &getState() override { return *this; }
+
+ /// See AbstractAttribute::getState(...).
+ const StateType &getState() const override { return *this; }
+};
+
+/// Helper class that provides common functionality to manifest IR attributes.
+template <Attribute::AttrKind AK, typename BaseType, typename AAType>
+struct IRAttribute : public BaseType {
+ IRAttribute(const IRPosition &IRP) : BaseType(IRP) {}
+
+ /// Most boolean IRAttribute AAs don't do anything non-trivial
+ /// in their initializers while non-boolean ones often do. Subclasses can
+ /// change this.
+ static bool hasTrivialInitializer() { return Attribute::isEnumAttrKind(AK); }
+
+ /// Compile time access to the IR attribute kind.
+ static constexpr Attribute::AttrKind IRAttributeKind = AK;
+
+ /// Return true if the IR attribute(s) associated with this AA are implied for
+ /// an undef value.
+ static bool isImpliedByUndef() { return true; }
+
+ /// Return true if the IR attribute(s) associated with this AA are implied for
+ /// an poison value.
+ static bool isImpliedByPoison() { return true; }
+
+ static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind = AK,
+ bool IgnoreSubsumingPositions = false) {
+ if (AAType::isImpliedByUndef() && isa<UndefValue>(IRP.getAssociatedValue()))
+ return true;
+ if (AAType::isImpliedByPoison() &&
+ isa<PoisonValue>(IRP.getAssociatedValue()))
+ return true;
+ return A.hasAttr(IRP, {ImpliedAttributeKind}, IgnoreSubsumingPositions,
+ ImpliedAttributeKind);
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ if (isa<UndefValue>(this->getIRPosition().getAssociatedValue()))
+ return ChangeStatus::UNCHANGED;
+ SmallVector<Attribute, 4> DeducedAttrs;
+ getDeducedAttributes(A, this->getAnchorValue().getContext(), DeducedAttrs);
+ if (DeducedAttrs.empty())
+ return ChangeStatus::UNCHANGED;
+ return A.manifestAttrs(this->getIRPosition(), DeducedAttrs);
+ }
+
+ /// Return the kind that identifies the abstract attribute implementation.
+ Attribute::AttrKind getAttrKind() const { return AK; }
+
+ /// Return the deduced attributes in \p Attrs.
+ virtual void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
+ SmallVectorImpl<Attribute> &Attrs) const {
+ Attrs.emplace_back(Attribute::get(Ctx, getAttrKind()));
+ }
+};
+
+/// Base struct for all "concrete attribute" deductions.
+///
+/// The abstract attribute is a minimal interface that allows the Attributor to
+/// orchestrate the abstract/fixpoint analysis. The design allows to hide away
+/// implementation choices made for the subclasses but also to structure their
+/// implementation and simplify the use of other abstract attributes in-flight.
+///
+/// To allow easy creation of new attributes, most methods have default
+/// implementations. The ones that do not are generally straight forward, except
+/// `AbstractAttribute::updateImpl` which is the location of most reasoning
+/// associated with the abstract attribute. The update is invoked by the
+/// Attributor in case the situation used to justify the current optimistic
+/// state might have changed. The Attributor determines this automatically
+/// by monitoring the `Attributor::getAAFor` calls made by abstract attributes.
+///
+/// The `updateImpl` method should inspect the IR and other abstract attributes
+/// in-flight to justify the best possible (=optimistic) state. The actual
+/// implementation is, similar to the underlying abstract state encoding, not
+/// exposed. In the most common case, the `updateImpl` will go through a list of
+/// reasons why its optimistic state is valid given the current information. If
+/// any combination of them holds and is sufficient to justify the current
+/// optimistic state, the method shall return UNCHAGED. If not, the optimistic
+/// state is adjusted to the situation and the method shall return CHANGED.
+///
+/// If the manifestation of the "concrete attribute" deduced by the subclass
+/// differs from the "default" behavior, which is a (set of) LLVM-IR
+/// attribute(s) for an argument, call site argument, function return value, or
+/// function, the `AbstractAttribute::manifest` method should be overloaded.
+///
+/// NOTE: If the state obtained via getState() is INVALID, thus if
+/// AbstractAttribute::getState().isValidState() returns false, no
+/// information provided by the methods of this class should be used.
+/// NOTE: The Attributor currently has certain limitations to what we can do.
+/// As a general rule of thumb, "concrete" abstract attributes should *for
+/// now* only perform "backward" information propagation. That means
+/// optimistic information obtained through abstract attributes should
+/// only be used at positions that precede the origin of the information
+/// with regards to the program flow. More practically, information can
+/// *now* be propagated from instructions to their enclosing function, but
+/// *not* from call sites to the called function. The mechanisms to allow
+/// both directions will be added in the future.
+/// NOTE: The mechanics of adding a new "concrete" abstract attribute are
+/// described in the file comment.
+struct AbstractAttribute : public IRPosition, public AADepGraphNode {
+ using StateType = AbstractState;
+
+ AbstractAttribute(const IRPosition &IRP) : IRPosition(IRP) {}
+
+ /// Virtual destructor.
+ virtual ~AbstractAttribute() = default;
+
+ /// Compile time access to the IR attribute kind.
+ static constexpr Attribute::AttrKind IRAttributeKind = Attribute::None;
+
+ /// This function is used to identify if an \p DGN is of type
+ /// AbstractAttribute so that the dyn_cast and cast can use such information
+ /// to cast an AADepGraphNode to an AbstractAttribute.
+ ///
+ /// We eagerly return true here because all AADepGraphNodes except for the
+ /// Synthethis Node are of type AbstractAttribute
+ static bool classof(const AADepGraphNode *DGN) { return true; }
+
+ /// Return false if this AA does anything non-trivial (hence not done by
+ /// default) in its initializer.
+ static bool hasTrivialInitializer() { return false; }
+
+ /// Return true if this AA requires a "callee" (or an associted function) for
+ /// a call site positon. Default is optimistic to minimize AAs.
+ static bool requiresCalleeForCallBase() { return false; }
+
+ /// Return true if this AA requires non-asm "callee" for a call site positon.
+ static bool requiresNonAsmForCallBase() { return true; }
+
+ /// Return true if this AA requires all callees for an argument or function
+ /// positon.
+ static bool requiresCallersForArgOrFunction() { return false; }
+
+ /// Return false if an AA should not be created for \p IRP.
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ return true;
+ }
+
+ /// Return false if an AA should not be updated for \p IRP.
+ static bool isValidIRPositionForUpdate(Attributor &A, const IRPosition &IRP) {
+ Function *AssociatedFn = IRP.getAssociatedFunction();
+ bool IsFnInterface = IRP.isFnInterfaceKind();
+ assert((!IsFnInterface || AssociatedFn) &&
+ "Function interface without a function?");
+
+ // TODO: Not all attributes require an exact definition. Find a way to
+ // enable deduction for some but not all attributes in case the
+ // definition might be changed at runtime, see also
+ // http://lists.llvm.org/pipermail/llvm-dev/2018-February/121275.html.
+ // TODO: We could always determine abstract attributes and if sufficient
+ // information was found we could duplicate the functions that do not
+ // have an exact definition.
+ return !IsFnInterface || A.isFunctionIPOAmendable(*AssociatedFn);
+ }
+
+ /// Initialize the state with the information in the Attributor \p A.
+ ///
+ /// This function is called by the Attributor once all abstract attributes
+ /// have been identified. It can and shall be used for task like:
+ /// - identify existing knowledge in the IR and use it for the "known state"
+ /// - perform any work that is not going to change over time, e.g., determine
+ /// a subset of the IR, or attributes in-flight, that have to be looked at
+ /// in the `updateImpl` method.
+ virtual void initialize(Attributor &A) {}
+
+ /// A query AA is always scheduled as long as we do updates because it does
+ /// lazy computation that cannot be determined to be done from the outside.
+ /// However, while query AAs will not be fixed if they do not have outstanding
+ /// dependences, we will only schedule them like other AAs. If a query AA that
+ /// received a new query it needs to request an update via
+ /// `Attributor::requestUpdateForAA`.
+ virtual bool isQueryAA() const { return false; }
+
+ /// Return the internal abstract state for inspection.
+ virtual StateType &getState() = 0;
+ virtual const StateType &getState() const = 0;
+
+ /// Return an IR position, see struct IRPosition.
+ const IRPosition &getIRPosition() const { return *this; };
+ IRPosition &getIRPosition() { return *this; };
+
+ /// Helper functions, for debug purposes only.
+ ///{
+ void print(raw_ostream &OS) const { print(nullptr, OS); }
+ void print(Attributor *, raw_ostream &OS) const override;
+ virtual void printWithDeps(raw_ostream &OS) const;
+ void dump() const { this->print(dbgs()); }
+
+ /// This function should return the "summarized" assumed state as string.
+ virtual const std::string getAsStr(Attributor *A) const = 0;
+
+ /// This function should return the name of the AbstractAttribute
+ virtual const std::string getName() const = 0;
+
+ /// This function should return the address of the ID of the AbstractAttribute
+ virtual const char *getIdAddr() const = 0;
+ ///}
+
+ /// Allow the Attributor access to the protected methods.
+ friend struct Attributor;
+
+protected:
+ /// Hook for the Attributor to trigger an update of the internal state.
+ ///
+ /// If this attribute is already fixed, this method will return UNCHANGED,
+ /// otherwise it delegates to `AbstractAttribute::updateImpl`.
+ ///
+ /// \Return CHANGED if the internal state changed, otherwise UNCHANGED.
+ ChangeStatus update(Attributor &A);
+
+ /// Hook for the Attributor to trigger the manifestation of the information
+ /// represented by the abstract attribute in the LLVM-IR.
+ ///
+ /// \Return CHANGED if the IR was altered, otherwise UNCHANGED.
+ virtual ChangeStatus manifest(Attributor &A) {
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// Hook to enable custom statistic tracking, called after manifest that
+ /// resulted in a change if statistics are enabled.
+ ///
+ /// We require subclasses to provide an implementation so we remember to
+ /// add statistics for them.
+ virtual void trackStatistics() const = 0;
+
+ /// The actual update/transfer function which has to be implemented by the
+ /// derived classes.
+ ///
+ /// If it is called, the environment has changed and we have to determine if
+ /// the current information is still valid or adjust it otherwise.
+ ///
+ /// \Return CHANGED if the internal state changed, otherwise UNCHANGED.
+ virtual ChangeStatus updateImpl(Attributor &A) = 0;
+};
+
+/// Forward declarations of output streams for debug purposes.
+///
+///{
+raw_ostream &operator<<(raw_ostream &OS, const AbstractAttribute &AA);
+raw_ostream &operator<<(raw_ostream &OS, ChangeStatus S);
+raw_ostream &operator<<(raw_ostream &OS, IRPosition::Kind);
+raw_ostream &operator<<(raw_ostream &OS, const IRPosition &);
+raw_ostream &operator<<(raw_ostream &OS, const AbstractState &State);
+template <typename base_ty, base_ty BestState, base_ty WorstState>
+raw_ostream &
+operator<<(raw_ostream &OS,
+ const IntegerStateBase<base_ty, BestState, WorstState> &S) {
+ return OS << "(" << S.getKnown() << "-" << S.getAssumed() << ")"
+ << static_cast<const AbstractState &>(S);
+}
+raw_ostream &operator<<(raw_ostream &OS, const IntegerRangeState &State);
+///}
+
+struct AttributorPass : public PassInfoMixin<AttributorPass> {
+ PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+struct AttributorCGSCCPass : public PassInfoMixin<AttributorCGSCCPass> {
+ PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
+ LazyCallGraph &CG, CGSCCUpdateResult &UR);
+};
+
+/// A more lightweight version of the Attributor which only runs attribute
+/// inference but no simplifications.
+struct AttributorLightPass : public PassInfoMixin<AttributorLightPass> {
+ PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+/// A more lightweight version of the Attributor which only runs attribute
+/// inference but no simplifications.
+struct AttributorLightCGSCCPass
+ : public PassInfoMixin<AttributorLightCGSCCPass> {
+ PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM,
+ LazyCallGraph &CG, CGSCCUpdateResult &UR);
+};
+
+/// Helper function to clamp a state \p S of type \p StateType with the
+/// information in \p R and indicate/return if \p S did change (as-in update is
+/// required to be run again).
+template <typename StateType>
+ChangeStatus clampStateAndIndicateChange(StateType &S, const StateType &R) {
+ auto Assumed = S.getAssumed();
+ S ^= R;
+ return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+}
+
+/// ----------------------------------------------------------------------------
+/// Abstract Attribute Classes
+/// ----------------------------------------------------------------------------
+
+struct AANoUnwind
+ : public IRAttribute<Attribute::NoUnwind,
+ StateWrapper<BooleanState, AbstractAttribute>,
+ AANoUnwind> {
+ AANoUnwind(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// Returns true if nounwind is assumed.
+ bool isAssumedNoUnwind() const { return getAssumed(); }
+
+ /// Returns true if nounwind is known.
+ bool isKnownNoUnwind() const { return getKnown(); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANoUnwind &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANoUnwind"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AANoUnwind
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+struct AANoSync
+ : public IRAttribute<Attribute::NoSync,
+ StateWrapper<BooleanState, AbstractAttribute>,
+ AANoSync> {
+ AANoSync(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions = false) {
+ // Note: This is also run for non-IPO amendable functions.
+ assert(ImpliedAttributeKind == Attribute::NoSync);
+ if (A.hasAttr(IRP, {Attribute::NoSync}, IgnoreSubsumingPositions,
+ Attribute::NoSync))
+ return true;
+
+ // Check for readonly + non-convergent.
+ // TODO: We should be able to use hasAttr for Attributes, not only
+ // AttrKinds.
+ Function *F = IRP.getAssociatedFunction();
+ if (!F || F->isConvergent())
+ return false;
+
+ SmallVector<Attribute, 2> Attrs;
+ A.getAttrs(IRP, {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
+
+ MemoryEffects ME = MemoryEffects::unknown();
+ for (const Attribute &Attr : Attrs)
+ ME &= Attr.getMemoryEffects();
+
+ if (!ME.onlyReadsMemory())
+ return false;
+
+ A.manifestAttrs(IRP, Attribute::get(F->getContext(), Attribute::NoSync));
+ return true;
+ }
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.isFunctionScope() &&
+ !IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return IRAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// Returns true if "nosync" is assumed.
+ bool isAssumedNoSync() const { return getAssumed(); }
+
+ /// Returns true if "nosync" is known.
+ bool isKnownNoSync() const { return getKnown(); }
+
+ /// Helper function used to determine whether an instruction is non-relaxed
+ /// atomic. In other words, if an atomic instruction does not have unordered
+ /// or monotonic ordering
+ static bool isNonRelaxedAtomic(const Instruction *I);
+
+ /// Helper function specific for intrinsics which are potentially volatile.
+ static bool isNoSyncIntrinsic(const Instruction *I);
+
+ /// Helper function to determine if \p CB is an aligned (GPU) barrier. Aligned
+ /// barriers have to be executed by all threads. The flag \p ExecutedAligned
+ /// indicates if the call is executed by all threads in a (thread) block in an
+ /// aligned way. If that is the case, non-aligned barriers are effectively
+ /// aligned barriers.
+ static bool isAlignedBarrier(const CallBase &CB, bool ExecutedAligned);
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANoSync &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANoSync"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AANoSync
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for all nonnull attributes.
+struct AAMustProgress
+ : public IRAttribute<Attribute::MustProgress,
+ StateWrapper<BooleanState, AbstractAttribute>,
+ AAMustProgress> {
+ AAMustProgress(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions = false) {
+ // Note: This is also run for non-IPO amendable functions.
+ assert(ImpliedAttributeKind == Attribute::MustProgress);
+ return A.hasAttr(IRP, {Attribute::MustProgress, Attribute::WillReturn},
+ IgnoreSubsumingPositions, Attribute::MustProgress);
+ }
+
+ /// Return true if we assume that the underlying value is nonnull.
+ bool isAssumedMustProgress() const { return getAssumed(); }
+
+ /// Return true if we know that underlying value is nonnull.
+ bool isKnownMustProgress() const { return getKnown(); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAMustProgress &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAMustProgress"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAMustProgress
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for all nonnull attributes.
+struct AANonNull
+ : public IRAttribute<Attribute::NonNull,
+ StateWrapper<BooleanState, AbstractAttribute>,
+ AANonNull> {
+ AANonNull(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// See AbstractAttribute::hasTrivialInitializer.
+ static bool hasTrivialInitializer() { return false; }
+
+ /// See IRAttribute::isImpliedByUndef.
+ /// Undef is not necessarily nonnull as nonnull + noundef would cause poison.
+ /// Poison implies nonnull though.
+ static bool isImpliedByUndef() { return false; }
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return IRAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// See AbstractAttribute::isImpliedByIR(...).
+ static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions = false);
+
+ /// Return true if we assume that the underlying value is nonnull.
+ bool isAssumedNonNull() const { return getAssumed(); }
+
+ /// Return true if we know that underlying value is nonnull.
+ bool isKnownNonNull() const { return getKnown(); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANonNull &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANonNull"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AANonNull
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract attribute for norecurse.
+struct AANoRecurse
+ : public IRAttribute<Attribute::NoRecurse,
+ StateWrapper<BooleanState, AbstractAttribute>,
+ AANoRecurse> {
+ AANoRecurse(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// Return true if "norecurse" is assumed.
+ bool isAssumedNoRecurse() const { return getAssumed(); }
+
+ /// Return true if "norecurse" is known.
+ bool isKnownNoRecurse() const { return getKnown(); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANoRecurse &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANoRecurse"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AANoRecurse
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract attribute for willreturn.
+struct AAWillReturn
+ : public IRAttribute<Attribute::WillReturn,
+ StateWrapper<BooleanState, AbstractAttribute>,
+ AAWillReturn> {
+ AAWillReturn(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions = false) {
+ // Note: This is also run for non-IPO amendable functions.
+ assert(ImpliedAttributeKind == Attribute::WillReturn);
+ if (IRAttribute::isImpliedByIR(A, IRP, ImpliedAttributeKind,
+ IgnoreSubsumingPositions))
+ return true;
+ if (!isImpliedByMustprogressAndReadonly(A, IRP))
+ return false;
+ A.manifestAttrs(IRP, Attribute::get(IRP.getAnchorValue().getContext(),
+ Attribute::WillReturn));
+ return true;
+ }
+
+ /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
+ static bool isImpliedByMustprogressAndReadonly(Attributor &A,
+ const IRPosition &IRP) {
+ // Check for `mustprogress` in the scope and the associated function which
+ // might be different if this is a call site.
+ if (!A.hasAttr(IRP, {Attribute::MustProgress}))
+ return false;
+
+ SmallVector<Attribute, 2> Attrs;
+ A.getAttrs(IRP, {Attribute::Memory}, Attrs,
+ /* IgnoreSubsumingPositions */ false);
+
+ MemoryEffects ME = MemoryEffects::unknown();
+ for (const Attribute &Attr : Attrs)
+ ME &= Attr.getMemoryEffects();
+ return ME.onlyReadsMemory();
+ }
+
+ /// Return true if "willreturn" is assumed.
+ bool isAssumedWillReturn() const { return getAssumed(); }
+
+ /// Return true if "willreturn" is known.
+ bool isKnownWillReturn() const { return getKnown(); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAWillReturn &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAWillReturn"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AAWillReturn
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract attribute for undefined behavior.
+struct AAUndefinedBehavior
+ : public StateWrapper<BooleanState, AbstractAttribute> {
+ using Base = StateWrapper<BooleanState, AbstractAttribute>;
+ AAUndefinedBehavior(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// Return true if "undefined behavior" is assumed.
+ bool isAssumedToCauseUB() const { return getAssumed(); }
+
+ /// Return true if "undefined behavior" is assumed for a specific instruction.
+ virtual bool isAssumedToCauseUB(Instruction *I) const = 0;
+
+ /// Return true if "undefined behavior" is known.
+ bool isKnownToCauseUB() const { return getKnown(); }
+
+ /// Return true if "undefined behavior" is known for a specific instruction.
+ virtual bool isKnownToCauseUB(Instruction *I) const = 0;
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAUndefinedBehavior &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAUndefinedBehavior"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAUndefineBehavior
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface to determine reachability of point A to B.
+struct AAIntraFnReachability
+ : public StateWrapper<BooleanState, AbstractAttribute> {
+ using Base = StateWrapper<BooleanState, AbstractAttribute>;
+ AAIntraFnReachability(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// Returns true if 'From' instruction is assumed to reach, 'To' instruction.
+ /// Users should provide two positions they are interested in, and the class
+ /// determines (and caches) reachability.
+ virtual bool isAssumedReachable(
+ Attributor &A, const Instruction &From, const Instruction &To,
+ const AA::InstExclusionSetTy *ExclusionSet = nullptr) const = 0;
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAIntraFnReachability &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAIntraFnReachability"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAIntraFnReachability
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for all noalias attributes.
+struct AANoAlias
+ : public IRAttribute<Attribute::NoAlias,
+ StateWrapper<BooleanState, AbstractAttribute>,
+ AANoAlias> {
+ AANoAlias(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return IRAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// See IRAttribute::isImpliedByIR
+ static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions = false);
+
+ /// See AbstractAttribute::requiresCallersForArgOrFunction
+ static bool requiresCallersForArgOrFunction() { return true; }
+
+ /// Return true if we assume that the underlying value is alias.
+ bool isAssumedNoAlias() const { return getAssumed(); }
+
+ /// Return true if we know that underlying value is noalias.
+ bool isKnownNoAlias() const { return getKnown(); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANoAlias &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANoAlias"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AANoAlias
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An AbstractAttribute for nofree.
+struct AANoFree
+ : public IRAttribute<Attribute::NoFree,
+ StateWrapper<BooleanState, AbstractAttribute>,
+ AANoFree> {
+ AANoFree(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// See IRAttribute::isImpliedByIR
+ static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions = false) {
+ // Note: This is also run for non-IPO amendable functions.
+ assert(ImpliedAttributeKind == Attribute::NoFree);
+ return A.hasAttr(
+ IRP, {Attribute::ReadNone, Attribute::ReadOnly, Attribute::NoFree},
+ IgnoreSubsumingPositions, Attribute::NoFree);
+ }
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.isFunctionScope() &&
+ !IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return IRAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// Return true if "nofree" is assumed.
+ bool isAssumedNoFree() const { return getAssumed(); }
+
+ /// Return true if "nofree" is known.
+ bool isKnownNoFree() const { return getKnown(); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANoFree &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANoFree"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AANoFree
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An AbstractAttribute for noreturn.
+struct AANoReturn
+ : public IRAttribute<Attribute::NoReturn,
+ StateWrapper<BooleanState, AbstractAttribute>,
+ AANoReturn> {
+ AANoReturn(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// Return true if the underlying object is assumed to never return.
+ bool isAssumedNoReturn() const { return getAssumed(); }
+
+ /// Return true if the underlying object is known to never return.
+ bool isKnownNoReturn() const { return getKnown(); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANoReturn &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANoReturn"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AANoReturn
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for liveness abstract attribute.
+struct AAIsDead
+ : public StateWrapper<BitIntegerState<uint8_t, 3, 0>, AbstractAttribute> {
+ using Base = StateWrapper<BitIntegerState<uint8_t, 3, 0>, AbstractAttribute>;
+ AAIsDead(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (IRP.getPositionKind() == IRPosition::IRP_FUNCTION)
+ return isa<Function>(IRP.getAnchorValue()) &&
+ !cast<Function>(IRP.getAnchorValue()).isDeclaration();
+ return true;
+ }
+
+ /// State encoding bits. A set bit in the state means the property holds.
+ enum {
+ HAS_NO_EFFECT = 1 << 0,
+ IS_REMOVABLE = 1 << 1,
+
+ IS_DEAD = HAS_NO_EFFECT | IS_REMOVABLE,
+ };
+ static_assert(IS_DEAD == getBestState(), "Unexpected BEST_STATE value");
+
+protected:
+ /// The query functions are protected such that other attributes need to go
+ /// through the Attributor interfaces: `Attributor::isAssumedDead(...)`
+
+ /// Returns true if the underlying value is assumed dead.
+ virtual bool isAssumedDead() const = 0;
+
+ /// Returns true if the underlying value is known dead.
+ virtual bool isKnownDead() const = 0;
+
+ /// Returns true if \p BB is known dead.
+ virtual bool isKnownDead(const BasicBlock *BB) const = 0;
+
+ /// Returns true if \p I is assumed dead.
+ virtual bool isAssumedDead(const Instruction *I) const = 0;
+
+ /// Returns true if \p I is known dead.
+ virtual bool isKnownDead(const Instruction *I) const = 0;
+
+ /// Return true if the underlying value is a store that is known to be
+ /// removable. This is different from dead stores as the removable store
+ /// can have an effect on live values, especially loads, but that effect
+ /// is propagated which allows us to remove the store in turn.
+ virtual bool isRemovableStore() const { return false; }
+
+ /// This method is used to check if at least one instruction in a collection
+ /// of instructions is live.
+ template <typename T> bool isLiveInstSet(T begin, T end) const {
+ for (const auto &I : llvm::make_range(begin, end)) {
+ assert(I->getFunction() == getIRPosition().getAssociatedFunction() &&
+ "Instruction must be in the same anchor scope function.");
+
+ if (!isAssumedDead(I))
+ return true;
+ }
+
+ return false;
+ }
+
+public:
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAIsDead &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// Determine if \p F might catch asynchronous exceptions.
+ static bool mayCatchAsynchronousExceptions(const Function &F) {
+ return F.hasPersonalityFn() && !canSimplifyInvokeNoUnwind(&F);
+ }
+
+ /// Returns true if \p BB is assumed dead.
+ virtual bool isAssumedDead(const BasicBlock *BB) const = 0;
+
+ /// Return if the edge from \p From BB to \p To BB is assumed dead.
+ /// This is specifically useful in AAReachability.
+ virtual bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const {
+ return false;
+ }
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAIsDead"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AAIsDead
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+
+ friend struct Attributor;
+};
+
+/// State for dereferenceable attribute
+struct DerefState : AbstractState {
+
+ static DerefState getBestState() { return DerefState(); }
+ static DerefState getBestState(const DerefState &) { return getBestState(); }
+
+ /// Return the worst possible representable state.
+ static DerefState getWorstState() {
+ DerefState DS;
+ DS.indicatePessimisticFixpoint();
+ return DS;
+ }
+ static DerefState getWorstState(const DerefState &) {
+ return getWorstState();
+ }
+
+ /// State representing for dereferenceable bytes.
+ IncIntegerState<> DerefBytesState;
+
+ /// Map representing for accessed memory offsets and sizes.
+ /// A key is Offset and a value is size.
+ /// If there is a load/store instruction something like,
+ /// p[offset] = v;
+ /// (offset, sizeof(v)) will be inserted to this map.
+ /// std::map is used because we want to iterate keys in ascending order.
+ std::map<int64_t, uint64_t> AccessedBytesMap;
+
+ /// Helper function to calculate dereferenceable bytes from current known
+ /// bytes and accessed bytes.
+ ///
+ /// int f(int *A){
+ /// *A = 0;
+ /// *(A+2) = 2;
+ /// *(A+1) = 1;
+ /// *(A+10) = 10;
+ /// }
+ /// ```
+ /// In that case, AccessedBytesMap is `{0:4, 4:4, 8:4, 40:4}`.
+ /// AccessedBytesMap is std::map so it is iterated in accending order on
+ /// key(Offset). So KnownBytes will be updated like this:
+ ///
+ /// |Access | KnownBytes
+ /// |(0, 4)| 0 -> 4
+ /// |(4, 4)| 4 -> 8
+ /// |(8, 4)| 8 -> 12
+ /// |(40, 4) | 12 (break)
+ void computeKnownDerefBytesFromAccessedMap() {
+ int64_t KnownBytes = DerefBytesState.getKnown();
+ for (auto &Access : AccessedBytesMap) {
+ if (KnownBytes < Access.first)
+ break;
+ KnownBytes = std::max(KnownBytes, Access.first + (int64_t)Access.second);
+ }
+
+ DerefBytesState.takeKnownMaximum(KnownBytes);
+ }
+
+ /// State representing that whether the value is globaly dereferenceable.
+ BooleanState GlobalState;
+
+ /// See AbstractState::isValidState()
+ bool isValidState() const override { return DerefBytesState.isValidState(); }
+
+ /// See AbstractState::isAtFixpoint()
+ bool isAtFixpoint() const override {
+ return !isValidState() ||
+ (DerefBytesState.isAtFixpoint() && GlobalState.isAtFixpoint());
+ }
+
+ /// See AbstractState::indicateOptimisticFixpoint(...)
+ ChangeStatus indicateOptimisticFixpoint() override {
+ DerefBytesState.indicateOptimisticFixpoint();
+ GlobalState.indicateOptimisticFixpoint();
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractState::indicatePessimisticFixpoint(...)
+ ChangeStatus indicatePessimisticFixpoint() override {
+ DerefBytesState.indicatePessimisticFixpoint();
+ GlobalState.indicatePessimisticFixpoint();
+ return ChangeStatus::CHANGED;
+ }
+
+ /// Update known dereferenceable bytes.
+ void takeKnownDerefBytesMaximum(uint64_t Bytes) {
+ DerefBytesState.takeKnownMaximum(Bytes);
+
+ // Known bytes might increase.
+ computeKnownDerefBytesFromAccessedMap();
+ }
+
+ /// Update assumed dereferenceable bytes.
+ void takeAssumedDerefBytesMinimum(uint64_t Bytes) {
+ DerefBytesState.takeAssumedMinimum(Bytes);
+ }
+
+ /// Add accessed bytes to the map.
+ void addAccessedBytes(int64_t Offset, uint64_t Size) {
+ uint64_t &AccessedBytes = AccessedBytesMap[Offset];
+ AccessedBytes = std::max(AccessedBytes, Size);
+
+ // Known bytes might increase.
+ computeKnownDerefBytesFromAccessedMap();
+ }
+
+ /// Equality for DerefState.
+ bool operator==(const DerefState &R) const {
+ return this->DerefBytesState == R.DerefBytesState &&
+ this->GlobalState == R.GlobalState;
+ }
+
+ /// Inequality for DerefState.
+ bool operator!=(const DerefState &R) const { return !(*this == R); }
+
+ /// See IntegerStateBase::operator^=
+ DerefState operator^=(const DerefState &R) {
+ DerefBytesState ^= R.DerefBytesState;
+ GlobalState ^= R.GlobalState;
+ return *this;
+ }
+
+ /// See IntegerStateBase::operator+=
+ DerefState operator+=(const DerefState &R) {
+ DerefBytesState += R.DerefBytesState;
+ GlobalState += R.GlobalState;
+ return *this;
+ }
+
+ /// See IntegerStateBase::operator&=
+ DerefState operator&=(const DerefState &R) {
+ DerefBytesState &= R.DerefBytesState;
+ GlobalState &= R.GlobalState;
+ return *this;
+ }
+
+ /// See IntegerStateBase::operator|=
+ DerefState operator|=(const DerefState &R) {
+ DerefBytesState |= R.DerefBytesState;
+ GlobalState |= R.GlobalState;
+ return *this;
+ }
+};
+
+/// An abstract interface for all dereferenceable attribute.
+struct AADereferenceable
+ : public IRAttribute<Attribute::Dereferenceable,
+ StateWrapper<DerefState, AbstractAttribute>,
+ AADereferenceable> {
+ AADereferenceable(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return IRAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// Return true if we assume that underlying value is
+ /// dereferenceable(_or_null) globally.
+ bool isAssumedGlobal() const { return GlobalState.getAssumed(); }
+
+ /// Return true if we know that underlying value is
+ /// dereferenceable(_or_null) globally.
+ bool isKnownGlobal() const { return GlobalState.getKnown(); }
+
+ /// Return assumed dereferenceable bytes.
+ uint32_t getAssumedDereferenceableBytes() const {
+ return DerefBytesState.getAssumed();
+ }
+
+ /// Return known dereferenceable bytes.
+ uint32_t getKnownDereferenceableBytes() const {
+ return DerefBytesState.getKnown();
+ }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AADereferenceable &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AADereferenceable"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AADereferenceable
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+using AAAlignmentStateType =
+ IncIntegerState<uint64_t, Value::MaximumAlignment, 1>;
+/// An abstract interface for all align attributes.
+struct AAAlign
+ : public IRAttribute<Attribute::Alignment,
+ StateWrapper<AAAlignmentStateType, AbstractAttribute>,
+ AAAlign> {
+ AAAlign(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return IRAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// Return assumed alignment.
+ Align getAssumedAlign() const { return Align(getAssumed()); }
+
+ /// Return known alignment.
+ Align getKnownAlign() const { return Align(getKnown()); }
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAAlign"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AAAlign
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAAlign &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface to track if a value leaves it's defining function
+/// instance.
+/// TODO: We should make it a ternary AA tracking uniqueness, and uniqueness
+/// wrt. the Attributor analysis separately.
+struct AAInstanceInfo : public StateWrapper<BooleanState, AbstractAttribute> {
+ AAInstanceInfo(const IRPosition &IRP, Attributor &A)
+ : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
+
+ /// Return true if we know that the underlying value is unique in its scope
+ /// wrt. the Attributor analysis. That means it might not be unique but we can
+ /// still use pointer equality without risking to represent two instances with
+ /// one `llvm::Value`.
+ bool isKnownUniqueForAnalysis() const { return isKnown(); }
+
+ /// Return true if we assume that the underlying value is unique in its scope
+ /// wrt. the Attributor analysis. That means it might not be unique but we can
+ /// still use pointer equality without risking to represent two instances with
+ /// one `llvm::Value`.
+ bool isAssumedUniqueForAnalysis() const { return isAssumed(); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAInstanceInfo &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAInstanceInfo"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAInstanceInfo
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for all nocapture attributes.
+struct AANoCapture
+ : public IRAttribute<
+ Attribute::NoCapture,
+ StateWrapper<BitIntegerState<uint16_t, 7, 0>, AbstractAttribute>,
+ AANoCapture> {
+ AANoCapture(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// See IRAttribute::isImpliedByIR
+ static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions = false);
+
+ /// Update \p State according to the capture capabilities of \p F for position
+ /// \p IRP.
+ static void determineFunctionCaptureCapabilities(const IRPosition &IRP,
+ const Function &F,
+ BitIntegerState &State);
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return IRAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// State encoding bits. A set bit in the state means the property holds.
+ /// NO_CAPTURE is the best possible state, 0 the worst possible state.
+ enum {
+ NOT_CAPTURED_IN_MEM = 1 << 0,
+ NOT_CAPTURED_IN_INT = 1 << 1,
+ NOT_CAPTURED_IN_RET = 1 << 2,
+
+ /// If we do not capture the value in memory or through integers we can only
+ /// communicate it back as a derived pointer.
+ NO_CAPTURE_MAYBE_RETURNED = NOT_CAPTURED_IN_MEM | NOT_CAPTURED_IN_INT,
+
+ /// If we do not capture the value in memory, through integers, or as a
+ /// derived pointer we know it is not captured.
+ NO_CAPTURE =
+ NOT_CAPTURED_IN_MEM | NOT_CAPTURED_IN_INT | NOT_CAPTURED_IN_RET,
+ };
+
+ /// Return true if we know that the underlying value is not captured in its
+ /// respective scope.
+ bool isKnownNoCapture() const { return isKnown(NO_CAPTURE); }
+
+ /// Return true if we assume that the underlying value is not captured in its
+ /// respective scope.
+ bool isAssumedNoCapture() const { return isAssumed(NO_CAPTURE); }
+
+ /// Return true if we know that the underlying value is not captured in its
+ /// respective scope but we allow it to escape through a "return".
+ bool isKnownNoCaptureMaybeReturned() const {
+ return isKnown(NO_CAPTURE_MAYBE_RETURNED);
+ }
+
+ /// Return true if we assume that the underlying value is not captured in its
+ /// respective scope but we allow it to escape through a "return".
+ bool isAssumedNoCaptureMaybeReturned() const {
+ return isAssumed(NO_CAPTURE_MAYBE_RETURNED);
+ }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANoCapture &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANoCapture"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AANoCapture
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+struct ValueSimplifyStateType : public AbstractState {
+
+ ValueSimplifyStateType(Type *Ty) : Ty(Ty) {}
+
+ static ValueSimplifyStateType getBestState(Type *Ty) {
+ return ValueSimplifyStateType(Ty);
+ }
+ static ValueSimplifyStateType getBestState(const ValueSimplifyStateType &VS) {
+ return getBestState(VS.Ty);
+ }
+
+ /// Return the worst possible representable state.
+ static ValueSimplifyStateType getWorstState(Type *Ty) {
+ ValueSimplifyStateType DS(Ty);
+ DS.indicatePessimisticFixpoint();
+ return DS;
+ }
+ static ValueSimplifyStateType
+ getWorstState(const ValueSimplifyStateType &VS) {
+ return getWorstState(VS.Ty);
+ }
+
+ /// See AbstractState::isValidState(...)
+ bool isValidState() const override { return BS.isValidState(); }
+
+ /// See AbstractState::isAtFixpoint(...)
+ bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
+
+ /// Return the assumed state encoding.
+ ValueSimplifyStateType getAssumed() { return *this; }
+ const ValueSimplifyStateType &getAssumed() const { return *this; }
+
+ /// See AbstractState::indicatePessimisticFixpoint(...)
+ ChangeStatus indicatePessimisticFixpoint() override {
+ return BS.indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractState::indicateOptimisticFixpoint(...)
+ ChangeStatus indicateOptimisticFixpoint() override {
+ return BS.indicateOptimisticFixpoint();
+ }
+
+ /// "Clamp" this state with \p PVS.
+ ValueSimplifyStateType operator^=(const ValueSimplifyStateType &VS) {
+ BS ^= VS.BS;
+ unionAssumed(VS.SimplifiedAssociatedValue);
+ return *this;
+ }
+
+ bool operator==(const ValueSimplifyStateType &RHS) const {
+ if (isValidState() != RHS.isValidState())
+ return false;
+ if (!isValidState() && !RHS.isValidState())
+ return true;
+ return SimplifiedAssociatedValue == RHS.SimplifiedAssociatedValue;
+ }
+
+protected:
+ /// The type of the original value.
+ Type *Ty;
+
+ /// Merge \p Other into the currently assumed simplified value
+ bool unionAssumed(std::optional<Value *> Other);
+
+ /// Helper to track validity and fixpoint
+ BooleanState BS;
+
+ /// An assumed simplified value. Initially, it is set to std::nullopt, which
+ /// means that the value is not clear under current assumption. If in the
+ /// pessimistic state, getAssumedSimplifiedValue doesn't return this value but
+ /// returns orignal associated value.
+ std::optional<Value *> SimplifiedAssociatedValue;
+};
+
+/// An abstract interface for value simplify abstract attribute.
+struct AAValueSimplify
+ : public StateWrapper<ValueSimplifyStateType, AbstractAttribute, Type *> {
+ using Base = StateWrapper<ValueSimplifyStateType, AbstractAttribute, Type *>;
+ AAValueSimplify(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, IRP.getAssociatedType()) {}
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAValueSimplify &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAValueSimplify"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAValueSimplify
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+
+private:
+ /// Return an assumed simplified value if a single candidate is found. If
+ /// there cannot be one, return original value. If it is not clear yet, return
+ /// std::nullopt.
+ ///
+ /// Use `Attributor::getAssumedSimplified` for value simplification.
+ virtual std::optional<Value *>
+ getAssumedSimplifiedValue(Attributor &A) const = 0;
+
+ friend struct Attributor;
+};
+
+struct AAHeapToStack : public StateWrapper<BooleanState, AbstractAttribute> {
+ using Base = StateWrapper<BooleanState, AbstractAttribute>;
+ AAHeapToStack(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// Returns true if HeapToStack conversion is assumed to be possible.
+ virtual bool isAssumedHeapToStack(const CallBase &CB) const = 0;
+
+ /// Returns true if HeapToStack conversion is assumed and the CB is a
+ /// callsite to a free operation to be removed.
+ virtual bool isAssumedHeapToStackRemovedFree(CallBase &CB) const = 0;
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAHeapToStack &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAHeapToStack"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AAHeapToStack
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for privatizability.
+///
+/// A pointer is privatizable if it can be replaced by a new, private one.
+/// Privatizing pointer reduces the use count, interaction between unrelated
+/// code parts.
+///
+/// In order for a pointer to be privatizable its value cannot be observed
+/// (=nocapture), it is (for now) not written (=readonly & noalias), we know
+/// what values are necessary to make the private copy look like the original
+/// one, and the values we need can be loaded (=dereferenceable).
+struct AAPrivatizablePtr
+ : public StateWrapper<BooleanState, AbstractAttribute> {
+ using Base = StateWrapper<BooleanState, AbstractAttribute>;
+ AAPrivatizablePtr(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return AbstractAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// Returns true if pointer privatization is assumed to be possible.
+ bool isAssumedPrivatizablePtr() const { return getAssumed(); }
+
+ /// Returns true if pointer privatization is known to be possible.
+ bool isKnownPrivatizablePtr() const { return getKnown(); }
+
+ /// See AbstractAttribute::requiresCallersForArgOrFunction
+ static bool requiresCallersForArgOrFunction() { return true; }
+
+ /// Return the type we can choose for a private copy of the underlying
+ /// value. std::nullopt means it is not clear yet, nullptr means there is
+ /// none.
+ virtual std::optional<Type *> getPrivatizableType() const = 0;
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAPrivatizablePtr &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAPrivatizablePtr"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAPricatizablePtr
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for memory access kind related attributes
+/// (readnone/readonly/writeonly).
+struct AAMemoryBehavior
+ : public IRAttribute<
+ Attribute::None,
+ StateWrapper<BitIntegerState<uint8_t, 3>, AbstractAttribute>,
+ AAMemoryBehavior> {
+ AAMemoryBehavior(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// See AbstractAttribute::hasTrivialInitializer.
+ static bool hasTrivialInitializer() { return false; }
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.isFunctionScope() &&
+ !IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return IRAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// State encoding bits. A set bit in the state means the property holds.
+ /// BEST_STATE is the best possible state, 0 the worst possible state.
+ enum {
+ NO_READS = 1 << 0,
+ NO_WRITES = 1 << 1,
+ NO_ACCESSES = NO_READS | NO_WRITES,
+
+ BEST_STATE = NO_ACCESSES,
+ };
+ static_assert(BEST_STATE == getBestState(), "Unexpected BEST_STATE value");
+
+ /// Return true if we know that the underlying value is not read or accessed
+ /// in its respective scope.
+ bool isKnownReadNone() const { return isKnown(NO_ACCESSES); }
+
+ /// Return true if we assume that the underlying value is not read or accessed
+ /// in its respective scope.
+ bool isAssumedReadNone() const { return isAssumed(NO_ACCESSES); }
+
+ /// Return true if we know that the underlying value is not accessed
+ /// (=written) in its respective scope.
+ bool isKnownReadOnly() const { return isKnown(NO_WRITES); }
+
+ /// Return true if we assume that the underlying value is not accessed
+ /// (=written) in its respective scope.
+ bool isAssumedReadOnly() const { return isAssumed(NO_WRITES); }
+
+ /// Return true if we know that the underlying value is not read in its
+ /// respective scope.
+ bool isKnownWriteOnly() const { return isKnown(NO_READS); }
+
+ /// Return true if we assume that the underlying value is not read in its
+ /// respective scope.
+ bool isAssumedWriteOnly() const { return isAssumed(NO_READS); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAMemoryBehavior &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAMemoryBehavior"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAMemoryBehavior
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for all memory location attributes
+/// (readnone/argmemonly/inaccessiblememonly/inaccessibleorargmemonly).
+struct AAMemoryLocation
+ : public IRAttribute<
+ Attribute::None,
+ StateWrapper<BitIntegerState<uint32_t, 511>, AbstractAttribute>,
+ AAMemoryLocation> {
+ using MemoryLocationsKind = StateType::base_t;
+
+ AAMemoryLocation(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// See AbstractAttribute::requiresCalleeForCallBase.
+ static bool requiresCalleeForCallBase() { return true; }
+
+ /// See AbstractAttribute::hasTrivialInitializer.
+ static bool hasTrivialInitializer() { return false; }
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.isFunctionScope() &&
+ !IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return IRAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// Encoding of different locations that could be accessed by a memory
+ /// access.
+ enum {
+ ALL_LOCATIONS = 0,
+ NO_LOCAL_MEM = 1 << 0,
+ NO_CONST_MEM = 1 << 1,
+ NO_GLOBAL_INTERNAL_MEM = 1 << 2,
+ NO_GLOBAL_EXTERNAL_MEM = 1 << 3,
+ NO_GLOBAL_MEM = NO_GLOBAL_INTERNAL_MEM | NO_GLOBAL_EXTERNAL_MEM,
+ NO_ARGUMENT_MEM = 1 << 4,
+ NO_INACCESSIBLE_MEM = 1 << 5,
+ NO_MALLOCED_MEM = 1 << 6,
+ NO_UNKOWN_MEM = 1 << 7,
+ NO_LOCATIONS = NO_LOCAL_MEM | NO_CONST_MEM | NO_GLOBAL_INTERNAL_MEM |
+ NO_GLOBAL_EXTERNAL_MEM | NO_ARGUMENT_MEM |
+ NO_INACCESSIBLE_MEM | NO_MALLOCED_MEM | NO_UNKOWN_MEM,
+
+ // Helper bit to track if we gave up or not.
+ VALID_STATE = NO_LOCATIONS + 1,
+
+ BEST_STATE = NO_LOCATIONS | VALID_STATE,
+ };
+ static_assert(BEST_STATE == getBestState(), "Unexpected BEST_STATE value");
+
+ /// Return true if we know that the associated functions has no observable
+ /// accesses.
+ bool isKnownReadNone() const { return isKnown(NO_LOCATIONS); }
+
+ /// Return true if we assume that the associated functions has no observable
+ /// accesses.
+ bool isAssumedReadNone() const {
+ return isAssumed(NO_LOCATIONS) || isAssumedStackOnly();
+ }
+
+ /// Return true if we know that the associated functions has at most
+ /// local/stack accesses.
+ bool isKnowStackOnly() const {
+ return isKnown(inverseLocation(NO_LOCAL_MEM, true, true));
+ }
+
+ /// Return true if we assume that the associated functions has at most
+ /// local/stack accesses.
+ bool isAssumedStackOnly() const {
+ return isAssumed(inverseLocation(NO_LOCAL_MEM, true, true));
+ }
+
+ /// Return true if we know that the underlying value will only access
+ /// inaccesible memory only (see Attribute::InaccessibleMemOnly).
+ bool isKnownInaccessibleMemOnly() const {
+ return isKnown(inverseLocation(NO_INACCESSIBLE_MEM, true, true));
+ }
+
+ /// Return true if we assume that the underlying value will only access
+ /// inaccesible memory only (see Attribute::InaccessibleMemOnly).
+ bool isAssumedInaccessibleMemOnly() const {
+ return isAssumed(inverseLocation(NO_INACCESSIBLE_MEM, true, true));
+ }
+
+ /// Return true if we know that the underlying value will only access
+ /// argument pointees (see Attribute::ArgMemOnly).
+ bool isKnownArgMemOnly() const {
+ return isKnown(inverseLocation(NO_ARGUMENT_MEM, true, true));
+ }
+
+ /// Return true if we assume that the underlying value will only access
+ /// argument pointees (see Attribute::ArgMemOnly).
+ bool isAssumedArgMemOnly() const {
+ return isAssumed(inverseLocation(NO_ARGUMENT_MEM, true, true));
+ }
+
+ /// Return true if we know that the underlying value will only access
+ /// inaccesible memory or argument pointees (see
+ /// Attribute::InaccessibleOrArgMemOnly).
+ bool isKnownInaccessibleOrArgMemOnly() const {
+ return isKnown(
+ inverseLocation(NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, true, true));
+ }
+
+ /// Return true if we assume that the underlying value will only access
+ /// inaccesible memory or argument pointees (see
+ /// Attribute::InaccessibleOrArgMemOnly).
+ bool isAssumedInaccessibleOrArgMemOnly() const {
+ return isAssumed(
+ inverseLocation(NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, true, true));
+ }
+
+ /// Return true if the underlying value may access memory through arguement
+ /// pointers of the associated function, if any.
+ bool mayAccessArgMem() const { return !isAssumed(NO_ARGUMENT_MEM); }
+
+ /// Return true if only the memory locations specififed by \p MLK are assumed
+ /// to be accessed by the associated function.
+ bool isAssumedSpecifiedMemOnly(MemoryLocationsKind MLK) const {
+ return isAssumed(MLK);
+ }
+
+ /// Return the locations that are assumed to be not accessed by the associated
+ /// function, if any.
+ MemoryLocationsKind getAssumedNotAccessedLocation() const {
+ return getAssumed();
+ }
+
+ /// Return the inverse of location \p Loc, thus for NO_XXX the return
+ /// describes ONLY_XXX. The flags \p AndLocalMem and \p AndConstMem determine
+ /// if local (=stack) and constant memory are allowed as well. Most of the
+ /// time we do want them to be included, e.g., argmemonly allows accesses via
+ /// argument pointers or local or constant memory accesses.
+ static MemoryLocationsKind
+ inverseLocation(MemoryLocationsKind Loc, bool AndLocalMem, bool AndConstMem) {
+ return NO_LOCATIONS & ~(Loc | (AndLocalMem ? NO_LOCAL_MEM : 0) |
+ (AndConstMem ? NO_CONST_MEM : 0));
+ };
+
+ /// Return the locations encoded by \p MLK as a readable string.
+ static std::string getMemoryLocationsAsStr(MemoryLocationsKind MLK);
+
+ /// Simple enum to distinguish read/write/read-write accesses.
+ enum AccessKind {
+ NONE = 0,
+ READ = 1 << 0,
+ WRITE = 1 << 1,
+ READ_WRITE = READ | WRITE,
+ };
+
+ /// Check \p Pred on all accesses to the memory kinds specified by \p MLK.
+ ///
+ /// This method will evaluate \p Pred on all accesses (access instruction +
+ /// underlying accessed memory pointer) and it will return true if \p Pred
+ /// holds every time.
+ virtual bool checkForAllAccessesToMemoryKind(
+ function_ref<bool(const Instruction *, const Value *, AccessKind,
+ MemoryLocationsKind)>
+ Pred,
+ MemoryLocationsKind MLK) const = 0;
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAMemoryLocation &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractState::getAsStr(Attributor).
+ const std::string getAsStr(Attributor *A) const override {
+ return getMemoryLocationsAsStr(getAssumedNotAccessedLocation());
+ }
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAMemoryLocation"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAMemoryLocation
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for range value analysis.
+struct AAValueConstantRange
+ : public StateWrapper<IntegerRangeState, AbstractAttribute, uint32_t> {
+ using Base = StateWrapper<IntegerRangeState, AbstractAttribute, uint32_t>;
+ AAValueConstantRange(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, IRP.getAssociatedType()->getIntegerBitWidth()) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isIntegerTy())
+ return false;
+ return AbstractAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// See AbstractAttribute::requiresCallersForArgOrFunction
+ static bool requiresCallersForArgOrFunction() { return true; }
+
+ /// See AbstractAttribute::getState(...).
+ IntegerRangeState &getState() override { return *this; }
+ const IntegerRangeState &getState() const override { return *this; }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAValueConstantRange &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// Return an assumed range for the associated value a program point \p CtxI.
+ /// If \p I is nullptr, simply return an assumed range.
+ virtual ConstantRange
+ getAssumedConstantRange(Attributor &A,
+ const Instruction *CtxI = nullptr) const = 0;
+
+ /// Return a known range for the associated value at a program point \p CtxI.
+ /// If \p I is nullptr, simply return a known range.
+ virtual ConstantRange
+ getKnownConstantRange(Attributor &A,
+ const Instruction *CtxI = nullptr) const = 0;
+
+ /// Return an assumed constant for the associated value a program point \p
+ /// CtxI.
+ std::optional<Constant *>
+ getAssumedConstant(Attributor &A, const Instruction *CtxI = nullptr) const {
+ ConstantRange RangeV = getAssumedConstantRange(A, CtxI);
+ if (auto *C = RangeV.getSingleElement()) {
+ Type *Ty = getAssociatedValue().getType();
+ return cast_or_null<Constant>(
+ AA::getWithType(*ConstantInt::get(Ty->getContext(), *C), *Ty));
+ }
+ if (RangeV.isEmptySet())
+ return std::nullopt;
+ return nullptr;
+ }
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAValueConstantRange"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAValueConstantRange
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// A class for a set state.
+/// The assumed boolean state indicates whether the corresponding set is full
+/// set or not. If the assumed state is false, this is the worst state. The
+/// worst state (invalid state) of set of potential values is when the set
+/// contains every possible value (i.e. we cannot in any way limit the value
+/// that the target position can take). That never happens naturally, we only
+/// force it. As for the conditions under which we force it, see
+/// AAPotentialConstantValues.
+template <typename MemberTy> struct PotentialValuesState : AbstractState {
+ using SetTy = SmallSetVector<MemberTy, 8>;
+
+ PotentialValuesState() : IsValidState(true), UndefIsContained(false) {}
+
+ PotentialValuesState(bool IsValid)
+ : IsValidState(IsValid), UndefIsContained(false) {}
+
+ /// See AbstractState::isValidState(...)
+ bool isValidState() const override { return IsValidState.isValidState(); }
+
+ /// See AbstractState::isAtFixpoint(...)
+ bool isAtFixpoint() const override { return IsValidState.isAtFixpoint(); }
+
+ /// See AbstractState::indicatePessimisticFixpoint(...)
+ ChangeStatus indicatePessimisticFixpoint() override {
+ return IsValidState.indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractState::indicateOptimisticFixpoint(...)
+ ChangeStatus indicateOptimisticFixpoint() override {
+ return IsValidState.indicateOptimisticFixpoint();
+ }
+
+ /// Return the assumed state
+ PotentialValuesState &getAssumed() { return *this; }
+ const PotentialValuesState &getAssumed() const { return *this; }
+
+ /// Return this set. We should check whether this set is valid or not by
+ /// isValidState() before calling this function.
+ const SetTy &getAssumedSet() const {
+ assert(isValidState() && "This set shoud not be used when it is invalid!");
+ return Set;
+ }
+
+ /// Returns whether this state contains an undef value or not.
+ bool undefIsContained() const {
+ assert(isValidState() && "This flag shoud not be used when it is invalid!");
+ return UndefIsContained;
+ }
+
+ bool operator==(const PotentialValuesState &RHS) const {
+ if (isValidState() != RHS.isValidState())
+ return false;
+ if (!isValidState() && !RHS.isValidState())
+ return true;
+ if (undefIsContained() != RHS.undefIsContained())
+ return false;
+ return Set == RHS.getAssumedSet();
+ }
+
+ /// Maximum number of potential values to be tracked.
+ /// This is set by -attributor-max-potential-values command line option
+ static unsigned MaxPotentialValues;
+
+ /// Return empty set as the best state of potential values.
+ static PotentialValuesState getBestState() {
+ return PotentialValuesState(true);
+ }
+
+ static PotentialValuesState getBestState(const PotentialValuesState &PVS) {
+ return getBestState();
+ }
+
+ /// Return full set as the worst state of potential values.
+ static PotentialValuesState getWorstState() {
+ return PotentialValuesState(false);
+ }
+
+ /// Union assumed set with the passed value.
+ void unionAssumed(const MemberTy &C) { insert(C); }
+
+ /// Union assumed set with assumed set of the passed state \p PVS.
+ void unionAssumed(const PotentialValuesState &PVS) { unionWith(PVS); }
+
+ /// Union assumed set with an undef value.
+ void unionAssumedWithUndef() { unionWithUndef(); }
+
+ /// "Clamp" this state with \p PVS.
+ PotentialValuesState operator^=(const PotentialValuesState &PVS) {
+ IsValidState ^= PVS.IsValidState;
+ unionAssumed(PVS);
+ return *this;
+ }
+
+ PotentialValuesState operator&=(const PotentialValuesState &PVS) {
+ IsValidState &= PVS.IsValidState;
+ unionAssumed(PVS);
+ return *this;
+ }
+
+ bool contains(const MemberTy &V) const {
+ return !isValidState() ? true : Set.contains(V);
+ }
+
+protected:
+ SetTy &getAssumedSet() {
+ assert(isValidState() && "This set shoud not be used when it is invalid!");
+ return Set;
+ }
+
+private:
+ /// Check the size of this set, and invalidate when the size is no
+ /// less than \p MaxPotentialValues threshold.
+ void checkAndInvalidate() {
+ if (Set.size() >= MaxPotentialValues)
+ indicatePessimisticFixpoint();
+ else
+ reduceUndefValue();
+ }
+
+ /// If this state contains both undef and not undef, we can reduce
+ /// undef to the not undef value.
+ void reduceUndefValue() { UndefIsContained = UndefIsContained & Set.empty(); }
+
+ /// Insert an element into this set.
+ void insert(const MemberTy &C) {
+ if (!isValidState())
+ return;
+ Set.insert(C);
+ checkAndInvalidate();
+ }
+
+ /// Take union with R.
+ void unionWith(const PotentialValuesState &R) {
+ /// If this is a full set, do nothing.
+ if (!isValidState())
+ return;
+ /// If R is full set, change L to a full set.
+ if (!R.isValidState()) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+ for (const MemberTy &C : R.Set)
+ Set.insert(C);
+ UndefIsContained |= R.undefIsContained();
+ checkAndInvalidate();
+ }
+
+ /// Take union with an undef value.
+ void unionWithUndef() {
+ UndefIsContained = true;
+ reduceUndefValue();
+ }
+
+ /// Take intersection with R.
+ void intersectWith(const PotentialValuesState &R) {
+ /// If R is a full set, do nothing.
+ if (!R.isValidState())
+ return;
+ /// If this is a full set, change this to R.
+ if (!isValidState()) {
+ *this = R;
+ return;
+ }
+ SetTy IntersectSet;
+ for (const MemberTy &C : Set) {
+ if (R.Set.count(C))
+ IntersectSet.insert(C);
+ }
+ Set = IntersectSet;
+ UndefIsContained &= R.undefIsContained();
+ reduceUndefValue();
+ }
+
+ /// A helper state which indicate whether this state is valid or not.
+ BooleanState IsValidState;
+
+ /// Container for potential values
+ SetTy Set;
+
+ /// Flag for undef value
+ bool UndefIsContained;
+};
+
+struct DenormalFPMathState : public AbstractState {
+ struct DenormalState {
+ DenormalMode Mode = DenormalMode::getInvalid();
+ DenormalMode ModeF32 = DenormalMode::getInvalid();
+
+ bool operator==(const DenormalState Other) const {
+ return Mode == Other.Mode && ModeF32 == Other.ModeF32;
+ }
+
+ bool operator!=(const DenormalState Other) const {
+ return Mode != Other.Mode || ModeF32 != Other.ModeF32;
+ }
+
+ bool isValid() const {
+ return Mode.isValid() && ModeF32.isValid();
+ }
+
+ static DenormalMode::DenormalModeKind
+ unionDenormalKind(DenormalMode::DenormalModeKind Callee,
+ DenormalMode::DenormalModeKind Caller) {
+ if (Caller == Callee)
+ return Caller;
+ if (Callee == DenormalMode::Dynamic)
+ return Caller;
+ if (Caller == DenormalMode::Dynamic)
+ return Callee;
+ return DenormalMode::Invalid;
+ }
+
+ static DenormalMode unionAssumed(DenormalMode Callee, DenormalMode Caller) {
+ return DenormalMode{unionDenormalKind(Callee.Output, Caller.Output),
+ unionDenormalKind(Callee.Input, Caller.Input)};
+ }
+
+ DenormalState unionWith(DenormalState Caller) const {
+ DenormalState Callee(*this);
+ Callee.Mode = unionAssumed(Callee.Mode, Caller.Mode);
+ Callee.ModeF32 = unionAssumed(Callee.ModeF32, Caller.ModeF32);
+ return Callee;
+ }
+ };
+
+ DenormalState Known;
+
+ /// Explicitly track whether we've hit a fixed point.
+ bool IsAtFixedpoint = false;
+
+ DenormalFPMathState() = default;
+
+ DenormalState getKnown() const { return Known; }
+
+ // There's only really known or unknown, there's no speculatively assumable
+ // state.
+ DenormalState getAssumed() const { return Known; }
+
+ bool isValidState() const override {
+ return Known.isValid();
+ }
+
+ /// Return true if there are no dynamic components to the denormal mode worth
+ /// specializing.
+ bool isModeFixed() const {
+ return Known.Mode.Input != DenormalMode::Dynamic &&
+ Known.Mode.Output != DenormalMode::Dynamic &&
+ Known.ModeF32.Input != DenormalMode::Dynamic &&
+ Known.ModeF32.Output != DenormalMode::Dynamic;
+ }
+
+ bool isAtFixpoint() const override {
+ return IsAtFixedpoint;
+ }
+
+ ChangeStatus indicateFixpoint() {
+ bool Changed = !IsAtFixedpoint;
+ IsAtFixedpoint = true;
+ return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
+ }
+
+ ChangeStatus indicateOptimisticFixpoint() override {
+ return indicateFixpoint();
+ }
+
+ ChangeStatus indicatePessimisticFixpoint() override {
+ return indicateFixpoint();
+ }
+
+ DenormalFPMathState operator^=(const DenormalFPMathState &Caller) {
+ Known = Known.unionWith(Caller.getKnown());
+ return *this;
+ }
+};
+
+using PotentialConstantIntValuesState = PotentialValuesState<APInt>;
+using PotentialLLVMValuesState =
+ PotentialValuesState<std::pair<AA::ValueAndContext, AA::ValueScope>>;
+
+raw_ostream &operator<<(raw_ostream &OS,
+ const PotentialConstantIntValuesState &R);
+raw_ostream &operator<<(raw_ostream &OS, const PotentialLLVMValuesState &R);
+
+/// An abstract interface for potential values analysis.
+///
+/// This AA collects potential values for each IR position.
+/// An assumed set of potential values is initialized with the empty set (the
+/// best state) and it will grow monotonically as we find more potential values
+/// for this position.
+/// The set might be forced to the worst state, that is, to contain every
+/// possible value for this position in 2 cases.
+/// 1. We surpassed the \p MaxPotentialValues threshold. This includes the
+/// case that this position is affected (e.g. because of an operation) by a
+/// Value that is in the worst state.
+/// 2. We tried to initialize on a Value that we cannot handle (e.g. an
+/// operator we do not currently handle).
+///
+/// For non constant integers see AAPotentialValues.
+struct AAPotentialConstantValues
+ : public StateWrapper<PotentialConstantIntValuesState, AbstractAttribute> {
+ using Base = StateWrapper<PotentialConstantIntValuesState, AbstractAttribute>;
+ AAPotentialConstantValues(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isIntegerTy())
+ return false;
+ return AbstractAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// See AbstractAttribute::requiresCallersForArgOrFunction
+ static bool requiresCallersForArgOrFunction() { return true; }
+
+ /// See AbstractAttribute::getState(...).
+ PotentialConstantIntValuesState &getState() override { return *this; }
+ const PotentialConstantIntValuesState &getState() const override {
+ return *this;
+ }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAPotentialConstantValues &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// Return assumed constant for the associated value
+ std::optional<Constant *>
+ getAssumedConstant(Attributor &A, const Instruction *CtxI = nullptr) const {
+ if (!isValidState())
+ return nullptr;
+ if (getAssumedSet().size() == 1) {
+ Type *Ty = getAssociatedValue().getType();
+ return cast_or_null<Constant>(AA::getWithType(
+ *ConstantInt::get(Ty->getContext(), *(getAssumedSet().begin())),
+ *Ty));
+ }
+ if (getAssumedSet().size() == 0) {
+ if (undefIsContained())
+ return UndefValue::get(getAssociatedValue().getType());
+ return std::nullopt;
+ }
+
+ return nullptr;
+ }
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override {
+ return "AAPotentialConstantValues";
+ }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAPotentialConstantValues
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+struct AAPotentialValues
+ : public StateWrapper<PotentialLLVMValuesState, AbstractAttribute> {
+ using Base = StateWrapper<PotentialLLVMValuesState, AbstractAttribute>;
+ AAPotentialValues(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// See AbstractAttribute::requiresCallersForArgOrFunction
+ static bool requiresCallersForArgOrFunction() { return true; }
+
+ /// See AbstractAttribute::getState(...).
+ PotentialLLVMValuesState &getState() override { return *this; }
+ const PotentialLLVMValuesState &getState() const override { return *this; }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAPotentialValues &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// Extract the single value in \p Values if any.
+ static Value *getSingleValue(Attributor &A, const AbstractAttribute &AA,
+ const IRPosition &IRP,
+ SmallVectorImpl<AA::ValueAndContext> &Values);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAPotentialValues"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAPotentialValues
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+
+private:
+ virtual bool getAssumedSimplifiedValues(
+ Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
+ AA::ValueScope, bool RecurseForSelectAndPHI = false) const = 0;
+
+ friend struct Attributor;
+};
+
+/// An abstract interface for all noundef attributes.
+struct AANoUndef
+ : public IRAttribute<Attribute::NoUndef,
+ StateWrapper<BooleanState, AbstractAttribute>,
+ AANoUndef> {
+ AANoUndef(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// See IRAttribute::isImpliedByUndef
+ static bool isImpliedByUndef() { return false; }
+
+ /// See IRAttribute::isImpliedByPoison
+ static bool isImpliedByPoison() { return false; }
+
+ /// See IRAttribute::isImpliedByIR
+ static bool isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions = false);
+
+ /// Return true if we assume that the underlying value is noundef.
+ bool isAssumedNoUndef() const { return getAssumed(); }
+
+ /// Return true if we know that underlying value is noundef.
+ bool isKnownNoUndef() const { return getKnown(); }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANoUndef &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANoUndef"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AANoUndef
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+struct AANoFPClass
+ : public IRAttribute<
+ Attribute::NoFPClass,
+ StateWrapper<BitIntegerState<uint32_t, fcAllFlags, fcNone>,
+ AbstractAttribute>,
+ AANoFPClass> {
+ using Base = StateWrapper<BitIntegerState<uint32_t, fcAllFlags, fcNone>,
+ AbstractAttribute>;
+
+ AANoFPClass(const IRPosition &IRP, Attributor &A) : IRAttribute(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ Type *Ty = IRP.getAssociatedType();
+ do {
+ if (Ty->isFPOrFPVectorTy())
+ return IRAttribute::isValidIRPositionForInit(A, IRP);
+ if (!Ty->isArrayTy())
+ break;
+ Ty = Ty->getArrayElementType();
+ } while (true);
+ return false;
+ }
+
+ /// Return true if we assume that the underlying value is nofpclass.
+ FPClassTest getAssumedNoFPClass() const {
+ return static_cast<FPClassTest>(getAssumed());
+ }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANoFPClass &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANoFPClass"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AANoFPClass
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+struct AACallGraphNode;
+struct AACallEdges;
+
+/// An Iterator for call edges, creates AACallEdges attributes in a lazy way.
+/// This iterator becomes invalid if the underlying edge list changes.
+/// So This shouldn't outlive a iteration of Attributor.
+class AACallEdgeIterator
+ : public iterator_adaptor_base<AACallEdgeIterator,
+ SetVector<Function *>::iterator> {
+ AACallEdgeIterator(Attributor &A, SetVector<Function *>::iterator Begin)
+ : iterator_adaptor_base(Begin), A(A) {}
+
+public:
+ AACallGraphNode *operator*() const;
+
+private:
+ Attributor &A;
+ friend AACallEdges;
+ friend AttributorCallGraph;
+};
+
+struct AACallGraphNode {
+ AACallGraphNode(Attributor &A) : A(A) {}
+ virtual ~AACallGraphNode() = default;
+
+ virtual AACallEdgeIterator optimisticEdgesBegin() const = 0;
+ virtual AACallEdgeIterator optimisticEdgesEnd() const = 0;
+
+ /// Iterator range for exploring the call graph.
+ iterator_range<AACallEdgeIterator> optimisticEdgesRange() const {
+ return iterator_range<AACallEdgeIterator>(optimisticEdgesBegin(),
+ optimisticEdgesEnd());
+ }
+
+protected:
+ /// Reference to Attributor needed for GraphTraits implementation.
+ Attributor &A;
+};
+
+/// An abstract state for querying live call edges.
+/// This interface uses the Attributor's optimistic liveness
+/// information to compute the edges that are alive.
+struct AACallEdges : public StateWrapper<BooleanState, AbstractAttribute>,
+ AACallGraphNode {
+ using Base = StateWrapper<BooleanState, AbstractAttribute>;
+
+ AACallEdges(const IRPosition &IRP, Attributor &A)
+ : Base(IRP), AACallGraphNode(A) {}
+
+ /// See AbstractAttribute::requiresNonAsmForCallBase.
+ static bool requiresNonAsmForCallBase() { return false; }
+
+ /// Get the optimistic edges.
+ virtual const SetVector<Function *> &getOptimisticEdges() const = 0;
+
+ /// Is there any call with a unknown callee.
+ virtual bool hasUnknownCallee() const = 0;
+
+ /// Is there any call with a unknown callee, excluding any inline asm.
+ virtual bool hasNonAsmUnknownCallee() const = 0;
+
+ /// Iterator for exploring the call graph.
+ AACallEdgeIterator optimisticEdgesBegin() const override {
+ return AACallEdgeIterator(A, getOptimisticEdges().begin());
+ }
+
+ /// Iterator for exploring the call graph.
+ AACallEdgeIterator optimisticEdgesEnd() const override {
+ return AACallEdgeIterator(A, getOptimisticEdges().end());
+ }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AACallEdges &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AACallEdges"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AACallEdges.
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+// Synthetic root node for the Attributor's internal call graph.
+struct AttributorCallGraph : public AACallGraphNode {
+ AttributorCallGraph(Attributor &A) : AACallGraphNode(A) {}
+ virtual ~AttributorCallGraph() = default;
+
+ AACallEdgeIterator optimisticEdgesBegin() const override {
+ return AACallEdgeIterator(A, A.Functions.begin());
+ }
+
+ AACallEdgeIterator optimisticEdgesEnd() const override {
+ return AACallEdgeIterator(A, A.Functions.end());
+ }
+
+ /// Force populate the entire call graph.
+ void populateAll() const {
+ for (const AACallGraphNode *AA : optimisticEdgesRange()) {
+ // Nothing else to do here.
+ (void)AA;
+ }
+ }
+
+ void print();
+};
+
+template <> struct GraphTraits<AACallGraphNode *> {
+ using NodeRef = AACallGraphNode *;
+ using ChildIteratorType = AACallEdgeIterator;
+
+ static AACallEdgeIterator child_begin(AACallGraphNode *Node) {
+ return Node->optimisticEdgesBegin();
+ }
+
+ static AACallEdgeIterator child_end(AACallGraphNode *Node) {
+ return Node->optimisticEdgesEnd();
+ }
+};
+
+template <>
+struct GraphTraits<AttributorCallGraph *>
+ : public GraphTraits<AACallGraphNode *> {
+ using nodes_iterator = AACallEdgeIterator;
+
+ static AACallGraphNode *getEntryNode(AttributorCallGraph *G) {
+ return static_cast<AACallGraphNode *>(G);
+ }
+
+ static AACallEdgeIterator nodes_begin(const AttributorCallGraph *G) {
+ return G->optimisticEdgesBegin();
+ }
+
+ static AACallEdgeIterator nodes_end(const AttributorCallGraph *G) {
+ return G->optimisticEdgesEnd();
+ }
+};
+
+template <>
+struct DOTGraphTraits<AttributorCallGraph *> : public DefaultDOTGraphTraits {
+ DOTGraphTraits(bool Simple = false) : DefaultDOTGraphTraits(Simple) {}
+
+ std::string getNodeLabel(const AACallGraphNode *Node,
+ const AttributorCallGraph *Graph) {
+ const AACallEdges *AACE = static_cast<const AACallEdges *>(Node);
+ return AACE->getAssociatedFunction()->getName().str();
+ }
+
+ static bool isNodeHidden(const AACallGraphNode *Node,
+ const AttributorCallGraph *Graph) {
+ // Hide the synth root.
+ return static_cast<const AACallGraphNode *>(Graph) == Node;
+ }
+};
+
+struct AAExecutionDomain
+ : public StateWrapper<BooleanState, AbstractAttribute> {
+ using Base = StateWrapper<BooleanState, AbstractAttribute>;
+ AAExecutionDomain(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// Summary about the execution domain of a block or instruction.
+ struct ExecutionDomainTy {
+ using BarriersSetTy = SmallPtrSet<CallBase *, 2>;
+ using AssumesSetTy = SmallPtrSet<AssumeInst *, 4>;
+
+ void addAssumeInst(Attributor &A, AssumeInst &AI) {
+ EncounteredAssumes.insert(&AI);
+ }
+
+ void addAlignedBarrier(Attributor &A, CallBase &CB) {
+ AlignedBarriers.insert(&CB);
+ }
+
+ void clearAssumeInstAndAlignedBarriers() {
+ EncounteredAssumes.clear();
+ AlignedBarriers.clear();
+ }
+
+ bool IsExecutedByInitialThreadOnly = true;
+ bool IsReachedFromAlignedBarrierOnly = true;
+ bool IsReachingAlignedBarrierOnly = true;
+ bool EncounteredNonLocalSideEffect = false;
+ BarriersSetTy AlignedBarriers;
+ AssumesSetTy EncounteredAssumes;
+ };
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAExecutionDomain &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName().
+ const std::string getName() const override { return "AAExecutionDomain"; }
+
+ /// See AbstractAttribute::getIdAddr().
+ const char *getIdAddr() const override { return &ID; }
+
+ /// Check if an instruction is executed only by the initial thread.
+ bool isExecutedByInitialThreadOnly(const Instruction &I) const {
+ return isExecutedByInitialThreadOnly(*I.getParent());
+ }
+
+ /// Check if a basic block is executed only by the initial thread.
+ virtual bool isExecutedByInitialThreadOnly(const BasicBlock &) const = 0;
+
+ /// Check if the instruction \p I is executed in an aligned region, that is,
+ /// the synchronizing effects before and after \p I are both aligned barriers.
+ /// This effectively means all threads execute \p I together.
+ virtual bool isExecutedInAlignedRegion(Attributor &A,
+ const Instruction &I) const = 0;
+
+ virtual ExecutionDomainTy getExecutionDomain(const BasicBlock &) const = 0;
+ /// Return the execution domain with which the call \p CB is entered and the
+ /// one with which it is left.
+ virtual std::pair<ExecutionDomainTy, ExecutionDomainTy>
+ getExecutionDomain(const CallBase &CB) const = 0;
+ virtual ExecutionDomainTy getFunctionExecutionDomain() const = 0;
+
+ /// Helper function to determine if \p FI is a no-op given the information
+ /// about its execution from \p ExecDomainAA.
+ virtual bool isNoOpFence(const FenceInst &FI) const = 0;
+
+ /// This function should return true if the type of the \p AA is
+ /// AAExecutionDomain.
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract Attribute for computing reachability between functions.
+struct AAInterFnReachability
+ : public StateWrapper<BooleanState, AbstractAttribute> {
+ using Base = StateWrapper<BooleanState, AbstractAttribute>;
+
+ AAInterFnReachability(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// If the function represented by this possition can reach \p Fn.
+ bool canReach(Attributor &A, const Function &Fn) const {
+ Function *Scope = getAnchorScope();
+ if (!Scope || Scope->isDeclaration())
+ return true;
+ return instructionCanReach(A, Scope->getEntryBlock().front(), Fn);
+ }
+
+ /// Can \p Inst reach \p Fn.
+ /// See also AA::isPotentiallyReachable.
+ virtual bool instructionCanReach(
+ Attributor &A, const Instruction &Inst, const Function &Fn,
+ const AA::InstExclusionSetTy *ExclusionSet = nullptr) const = 0;
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAInterFnReachability &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAInterFnReachability"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is AACallEdges.
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract Attribute for determining the necessity of the convergent
+/// attribute.
+struct AANonConvergent : public StateWrapper<BooleanState, AbstractAttribute> {
+ using Base = StateWrapper<BooleanState, AbstractAttribute>;
+
+ AANonConvergent(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AANonConvergent &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// Return true if "non-convergent" is assumed.
+ bool isAssumedNotConvergent() const { return getAssumed(); }
+
+ /// Return true if "non-convergent" is known.
+ bool isKnownNotConvergent() const { return getKnown(); }
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AANonConvergent"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AANonConvergent.
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for struct information.
+struct AAPointerInfo : public AbstractAttribute {
+ AAPointerInfo(const IRPosition &IRP) : AbstractAttribute(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return AbstractAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ enum AccessKind {
+ // First two bits to distinguish may and must accesses.
+ AK_MUST = 1 << 0,
+ AK_MAY = 1 << 1,
+
+ // Then two bits for read and write. These are not exclusive.
+ AK_R = 1 << 2,
+ AK_W = 1 << 3,
+ AK_RW = AK_R | AK_W,
+
+ // One special case for assumptions about memory content. These
+ // are neither reads nor writes. They are however always modeled
+ // as read to avoid using them for write removal.
+ AK_ASSUMPTION = (1 << 4) | AK_MUST,
+
+ // Helper for easy access.
+ AK_MAY_READ = AK_MAY | AK_R,
+ AK_MAY_WRITE = AK_MAY | AK_W,
+ AK_MAY_READ_WRITE = AK_MAY | AK_R | AK_W,
+ AK_MUST_READ = AK_MUST | AK_R,
+ AK_MUST_WRITE = AK_MUST | AK_W,
+ AK_MUST_READ_WRITE = AK_MUST | AK_R | AK_W,
+ };
+
+ /// A container for a list of ranges.
+ struct RangeList {
+ // The set of ranges rarely contains more than one element, and is unlikely
+ // to contain more than say four elements. So we find the middle-ground with
+ // a sorted vector. This avoids hard-coding a rarely used number like "four"
+ // into every instance of a SmallSet.
+ using RangeTy = AA::RangeTy;
+ using VecTy = SmallVector<RangeTy>;
+ using iterator = VecTy::iterator;
+ using const_iterator = VecTy::const_iterator;
+ VecTy Ranges;
+
+ RangeList(const RangeTy &R) { Ranges.push_back(R); }
+ RangeList(ArrayRef<int64_t> Offsets, int64_t Size) {
+ Ranges.reserve(Offsets.size());
+ for (unsigned i = 0, e = Offsets.size(); i != e; ++i) {
+ assert(((i + 1 == e) || Offsets[i] < Offsets[i + 1]) &&
+ "Expected strictly ascending offsets.");
+ Ranges.emplace_back(Offsets[i], Size);
+ }
+ }
+ RangeList() = default;
+
+ iterator begin() { return Ranges.begin(); }
+ iterator end() { return Ranges.end(); }
+ const_iterator begin() const { return Ranges.begin(); }
+ const_iterator end() const { return Ranges.end(); }
+
+ // Helpers required for std::set_difference
+ using value_type = RangeTy;
+ void push_back(const RangeTy &R) {
+ assert((Ranges.empty() || RangeTy::OffsetLessThan(Ranges.back(), R)) &&
+ "Ensure the last element is the greatest.");
+ Ranges.push_back(R);
+ }
+
+ /// Copy ranges from \p L that are not in \p R, into \p D.
+ static void set_difference(const RangeList &L, const RangeList &R,
+ RangeList &D) {
+ std::set_difference(L.begin(), L.end(), R.begin(), R.end(),
+ std::back_inserter(D), RangeTy::OffsetLessThan);
+ }
+
+ unsigned size() const { return Ranges.size(); }
+
+ bool operator==(const RangeList &OI) const { return Ranges == OI.Ranges; }
+
+ /// Merge the ranges in \p RHS into the current ranges.
+ /// - Merging a list of unknown ranges makes the current list unknown.
+ /// - Ranges with the same offset are merged according to RangeTy::operator&
+ /// \return true if the current RangeList changed.
+ bool merge(const RangeList &RHS) {
+ if (isUnknown())
+ return false;
+ if (RHS.isUnknown()) {
+ setUnknown();
+ return true;
+ }
+
+ if (Ranges.empty()) {
+ Ranges = RHS.Ranges;
+ return true;
+ }
+
+ bool Changed = false;
+ auto LPos = Ranges.begin();
+ for (auto &R : RHS.Ranges) {
+ auto Result = insert(LPos, R);
+ if (isUnknown())
+ return true;
+ LPos = Result.first;
+ Changed |= Result.second;
+ }
+ return Changed;
+ }
+
+ /// Insert \p R at the given iterator \p Pos, and merge if necessary.
+ ///
+ /// This assumes that all ranges before \p Pos are OffsetLessThan \p R, and
+ /// then maintains the sorted order for the suffix list.
+ ///
+ /// \return The place of insertion and true iff anything changed.
+ std::pair<iterator, bool> insert(iterator Pos, const RangeTy &R) {
+ if (isUnknown())
+ return std::make_pair(Ranges.begin(), false);
+ if (R.offsetOrSizeAreUnknown()) {
+ return std::make_pair(setUnknown(), true);
+ }
+
+ // Maintain this as a sorted vector of unique entries.
+ auto LB = std::lower_bound(Pos, Ranges.end(), R, RangeTy::OffsetLessThan);
+ if (LB == Ranges.end() || LB->Offset != R.Offset)
+ return std::make_pair(Ranges.insert(LB, R), true);
+ bool Changed = *LB != R;
+ *LB &= R;
+ if (LB->offsetOrSizeAreUnknown())
+ return std::make_pair(setUnknown(), true);
+ return std::make_pair(LB, Changed);
+ }
+
+ /// Insert the given range \p R, maintaining sorted order.
+ ///
+ /// \return The place of insertion and true iff anything changed.
+ std::pair<iterator, bool> insert(const RangeTy &R) {
+ return insert(Ranges.begin(), R);
+ }
+
+ /// Add the increment \p Inc to the offset of every range.
+ void addToAllOffsets(int64_t Inc) {
+ assert(!isUnassigned() &&
+ "Cannot increment if the offset is not yet computed!");
+ if (isUnknown())
+ return;
+ for (auto &R : Ranges) {
+ R.Offset += Inc;
+ }
+ }
+
+ /// Return true iff there is exactly one range and it is known.
+ bool isUnique() const {
+ return Ranges.size() == 1 && !Ranges.front().offsetOrSizeAreUnknown();
+ }
+
+ /// Return the unique range, assuming it exists.
+ const RangeTy &getUnique() const {
+ assert(isUnique() && "No unique range to return!");
+ return Ranges.front();
+ }
+
+ /// Return true iff the list contains an unknown range.
+ bool isUnknown() const {
+ if (isUnassigned())
+ return false;
+ if (Ranges.front().offsetOrSizeAreUnknown()) {
+ assert(Ranges.size() == 1 && "Unknown is a singleton range.");
+ return true;
+ }
+ return false;
+ }
+
+ /// Discard all ranges and insert a single unknown range.
+ iterator setUnknown() {
+ Ranges.clear();
+ Ranges.push_back(RangeTy::getUnknown());
+ return Ranges.begin();
+ }
+
+ /// Return true if no ranges have been inserted.
+ bool isUnassigned() const { return Ranges.size() == 0; }
+ };
+
+ /// An access description.
+ struct Access {
+ Access(Instruction *I, int64_t Offset, int64_t Size,
+ std::optional<Value *> Content, AccessKind Kind, Type *Ty)
+ : LocalI(I), RemoteI(I), Content(Content), Ranges(Offset, Size),
+ Kind(Kind), Ty(Ty) {
+ verify();
+ }
+ Access(Instruction *LocalI, Instruction *RemoteI, const RangeList &Ranges,
+ std::optional<Value *> Content, AccessKind K, Type *Ty)
+ : LocalI(LocalI), RemoteI(RemoteI), Content(Content), Ranges(Ranges),
+ Kind(K), Ty(Ty) {
+ if (Ranges.size() > 1) {
+ Kind = AccessKind(Kind | AK_MAY);
+ Kind = AccessKind(Kind & ~AK_MUST);
+ }
+ verify();
+ }
+ Access(Instruction *LocalI, Instruction *RemoteI, int64_t Offset,
+ int64_t Size, std::optional<Value *> Content, AccessKind Kind,
+ Type *Ty)
+ : LocalI(LocalI), RemoteI(RemoteI), Content(Content),
+ Ranges(Offset, Size), Kind(Kind), Ty(Ty) {
+ verify();
+ }
+ Access(const Access &Other) = default;
+
+ Access &operator=(const Access &Other) = default;
+ bool operator==(const Access &R) const {
+ return LocalI == R.LocalI && RemoteI == R.RemoteI && Ranges == R.Ranges &&
+ Content == R.Content && Kind == R.Kind;
+ }
+ bool operator!=(const Access &R) const { return !(*this == R); }
+
+ Access &operator&=(const Access &R) {
+ assert(RemoteI == R.RemoteI && "Expected same instruction!");
+ assert(LocalI == R.LocalI && "Expected same instruction!");
+
+ // Note that every Access object corresponds to a unique Value, and only
+ // accesses to the same Value are merged. Hence we assume that all ranges
+ // are the same size. If ranges can be different size, then the contents
+ // must be dropped.
+ Ranges.merge(R.Ranges);
+ Content =
+ AA::combineOptionalValuesInAAValueLatice(Content, R.Content, Ty);
+
+ // Combine the access kind, which results in a bitwise union.
+ // If there is more than one range, then this must be a MAY.
+ // If we combine a may and a must access we clear the must bit.
+ Kind = AccessKind(Kind | R.Kind);
+ if ((Kind & AK_MAY) || Ranges.size() > 1) {
+ Kind = AccessKind(Kind | AK_MAY);
+ Kind = AccessKind(Kind & ~AK_MUST);
+ }
+ verify();
+ return *this;
+ }
+
+ void verify() {
+ assert(isMustAccess() + isMayAccess() == 1 &&
+ "Expect must or may access, not both.");
+ assert(isAssumption() + isWrite() <= 1 &&
+ "Expect assumption access or write access, never both.");
+ assert((isMayAccess() || Ranges.size() == 1) &&
+ "Cannot be a must access if there are multiple ranges.");
+ }
+
+ /// Return the access kind.
+ AccessKind getKind() const { return Kind; }
+
+ /// Return true if this is a read access.
+ bool isRead() const { return Kind & AK_R; }
+
+ /// Return true if this is a write access.
+ bool isWrite() const { return Kind & AK_W; }
+
+ /// Return true if this is a write access.
+ bool isWriteOrAssumption() const { return isWrite() || isAssumption(); }
+
+ /// Return true if this is an assumption access.
+ bool isAssumption() const { return Kind == AK_ASSUMPTION; }
+
+ bool isMustAccess() const {
+ bool MustAccess = Kind & AK_MUST;
+ assert((!MustAccess || Ranges.size() < 2) &&
+ "Cannot be a must access if there are multiple ranges.");
+ return MustAccess;
+ }
+
+ bool isMayAccess() const {
+ bool MayAccess = Kind & AK_MAY;
+ assert((MayAccess || Ranges.size() < 2) &&
+ "Cannot be a must access if there are multiple ranges.");
+ return MayAccess;
+ }
+
+ /// Return the instruction that causes the access with respect to the local
+ /// scope of the associated attribute.
+ Instruction *getLocalInst() const { return LocalI; }
+
+ /// Return the actual instruction that causes the access.
+ Instruction *getRemoteInst() const { return RemoteI; }
+
+ /// Return true if the value written is not known yet.
+ bool isWrittenValueYetUndetermined() const { return !Content; }
+
+ /// Return true if the value written cannot be determined at all.
+ bool isWrittenValueUnknown() const {
+ return Content.has_value() && !*Content;
+ }
+
+ /// Set the value written to nullptr, i.e., unknown.
+ void setWrittenValueUnknown() { Content = nullptr; }
+
+ /// Return the type associated with the access, if known.
+ Type *getType() const { return Ty; }
+
+ /// Return the value writen, if any.
+ Value *getWrittenValue() const {
+ assert(!isWrittenValueYetUndetermined() &&
+ "Value needs to be determined before accessing it.");
+ return *Content;
+ }
+
+ /// Return the written value which can be `llvm::null` if it is not yet
+ /// determined.
+ std::optional<Value *> getContent() const { return Content; }
+
+ bool hasUniqueRange() const { return Ranges.isUnique(); }
+ const AA::RangeTy &getUniqueRange() const { return Ranges.getUnique(); }
+
+ /// Add a range accessed by this Access.
+ ///
+ /// If there are multiple ranges, then this is a "may access".
+ void addRange(int64_t Offset, int64_t Size) {
+ Ranges.insert({Offset, Size});
+ if (!hasUniqueRange()) {
+ Kind = AccessKind(Kind | AK_MAY);
+ Kind = AccessKind(Kind & ~AK_MUST);
+ }
+ }
+
+ const RangeList &getRanges() const { return Ranges; }
+
+ using const_iterator = RangeList::const_iterator;
+ const_iterator begin() const { return Ranges.begin(); }
+ const_iterator end() const { return Ranges.end(); }
+
+ private:
+ /// The instruction responsible for the access with respect to the local
+ /// scope of the associated attribute.
+ Instruction *LocalI;
+
+ /// The instruction responsible for the access.
+ Instruction *RemoteI;
+
+ /// The value written, if any. `std::nullopt` means "not known yet",
+ /// `nullptr` cannot be determined.
+ std::optional<Value *> Content;
+
+ /// Set of potential ranges accessed from the base pointer.
+ RangeList Ranges;
+
+ /// The access kind, e.g., READ, as bitset (could be more than one).
+ AccessKind Kind;
+
+ /// The type of the content, thus the type read/written, can be null if not
+ /// available.
+ Type *Ty;
+ };
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAPointerInfo &createForPosition(const IRPosition &IRP, Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAPointerInfo"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ using OffsetBinsTy = DenseMap<AA::RangeTy, SmallSet<unsigned, 4>>;
+ using const_bin_iterator = OffsetBinsTy::const_iterator;
+ virtual const_bin_iterator begin() const = 0;
+ virtual const_bin_iterator end() const = 0;
+ virtual int64_t numOffsetBins() const = 0;
+ virtual void dumpState(raw_ostream &O) const = 0;
+ virtual const Access &getBinAccess(unsigned Index) const = 0;
+
+ /// Call \p CB on all accesses that might interfere with \p Range and return
+ /// true if all such accesses were known and the callback returned true for
+ /// all of them, false otherwise. An access interferes with an offset-size
+ /// pair if it might read or write that memory region.
+ virtual bool forallInterferingAccesses(
+ AA::RangeTy Range, function_ref<bool(const Access &, bool)> CB) const = 0;
+
+ /// Call \p CB on all accesses that might interfere with \p I and
+ /// return true if all such accesses were known and the callback returned true
+ /// for all of them, false otherwise. In contrast to forallInterferingAccesses
+ /// this function will perform reasoning to exclude write accesses that cannot
+ /// affect the load even if they on the surface look as if they would. The
+ /// flag \p HasBeenWrittenTo will be set to true if we know that \p I does not
+ /// read the initial value of the underlying memory. If \p SkipCB is given and
+ /// returns false for a potentially interfering access, that access is not
+ /// checked for actual interference.
+ virtual bool forallInterferingAccesses(
+ Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
+ bool FindInterferingWrites, bool FindInterferingReads,
+ function_ref<bool(const Access &, bool)> CB, bool &HasBeenWrittenTo,
+ AA::RangeTy &Range,
+ function_ref<bool(const Access &)> SkipCB = nullptr) const = 0;
+
+ /// This function should return true if the type of the \p AA is AAPointerInfo
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+raw_ostream &operator<<(raw_ostream &, const AAPointerInfo::Access &);
+
+/// An abstract attribute for getting assumption information.
+struct AAAssumptionInfo
+ : public StateWrapper<SetState<StringRef>, AbstractAttribute,
+ DenseSet<StringRef>> {
+ using Base =
+ StateWrapper<SetState<StringRef>, AbstractAttribute, DenseSet<StringRef>>;
+
+ AAAssumptionInfo(const IRPosition &IRP, Attributor &A,
+ const DenseSet<StringRef> &Known)
+ : Base(IRP, Known) {}
+
+ /// Returns true if the assumption set contains the assumption \p Assumption.
+ virtual bool hasAssumption(const StringRef Assumption) const = 0;
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAAssumptionInfo &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAAssumptionInfo"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAAssumptionInfo
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract attribute for getting all assumption underlying objects.
+struct AAUnderlyingObjects : AbstractAttribute {
+ AAUnderlyingObjects(const IRPosition &IRP) : AbstractAttribute(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return AbstractAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// See AbstractAttribute::requiresCallersForArgOrFunction
+ static bool requiresCallersForArgOrFunction() { return true; }
+
+ /// Create an abstract attribute biew for the position \p IRP.
+ static AAUnderlyingObjects &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAUnderlyingObjects"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAUnderlyingObjects.
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+
+ /// Check \p Pred on all underlying objects in \p Scope collected so far.
+ ///
+ /// This method will evaluate \p Pred on all underlying objects in \p Scope
+ /// collected so far and return true if \p Pred holds on all of them.
+ virtual bool
+ forallUnderlyingObjects(function_ref<bool(Value &)> Pred,
+ AA::ValueScope Scope = AA::Interprocedural) const = 0;
+};
+
+/// An abstract interface for address space information.
+struct AAAddressSpace : public StateWrapper<BooleanState, AbstractAttribute> {
+ AAAddressSpace(const IRPosition &IRP, Attributor &A)
+ : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return AbstractAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// See AbstractAttribute::requiresCallersForArgOrFunction
+ static bool requiresCallersForArgOrFunction() { return true; }
+
+ /// Return the address space of the associated value. \p NoAddressSpace is
+ /// returned if the associated value is dead. This functions is not supposed
+ /// to be called if the AA is invalid.
+ virtual int32_t getAddressSpace() const = 0;
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAAddressSpace &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAAddressSpace"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAAssumptionInfo
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ // No address space which indicates the associated value is dead.
+ static const int32_t NoAddressSpace = -1;
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+struct AAAllocationInfo : public StateWrapper<BooleanState, AbstractAttribute> {
+ AAAllocationInfo(const IRPosition &IRP, Attributor &A)
+ : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (!IRP.getAssociatedType()->isPtrOrPtrVectorTy())
+ return false;
+ return AbstractAttribute::isValidIRPositionForInit(A, IRP);
+ }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAAllocationInfo &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ virtual std::optional<TypeSize> getAllocatedSize() const = 0;
+
+ using NewOffsetsTy = DenseMap<AA::RangeTy, AA::RangeTy>;
+ virtual const NewOffsetsTy &getNewOffsets() const = 0;
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAAllocationInfo"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAAllocationInfo
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ constexpr static const std::optional<TypeSize> HasNoAllocationSize =
+ std::optional<TypeSize>(TypeSize(-1, true));
+
+ static const char ID;
+};
+
+/// An abstract interface for llvm::GlobalValue information interference.
+struct AAGlobalValueInfo
+ : public StateWrapper<BooleanState, AbstractAttribute> {
+ AAGlobalValueInfo(const IRPosition &IRP, Attributor &A)
+ : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (IRP.getPositionKind() != IRPosition::IRP_FLOAT)
+ return false;
+ auto *GV = dyn_cast<GlobalValue>(&IRP.getAnchorValue());
+ if (!GV)
+ return false;
+ return GV->hasLocalLinkage();
+ }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAGlobalValueInfo &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// Return true iff \p U is a potential use of the associated global value.
+ virtual bool isPotentialUse(const Use &U) const = 0;
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAGlobalValueInfo"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAGlobalValueInfo
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract interface for indirect call information interference.
+struct AAIndirectCallInfo
+ : public StateWrapper<BooleanState, AbstractAttribute> {
+ AAIndirectCallInfo(const IRPosition &IRP, Attributor &A)
+ : StateWrapper<BooleanState, AbstractAttribute>(IRP) {}
+
+ /// See AbstractAttribute::isValidIRPositionForInit
+ static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
+ if (IRP.getPositionKind() != IRPosition::IRP_CALL_SITE)
+ return false;
+ auto *CB = cast<CallBase>(IRP.getCtxI());
+ return CB->getOpcode() == Instruction::Call && CB->isIndirectCall() &&
+ !CB->isMustTailCall();
+ }
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AAIndirectCallInfo &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// Call \CB on each potential callee value and return true if all were known
+ /// and \p CB returned true on all of them. Otherwise, return false.
+ virtual bool foreachCallee(function_ref<bool(Function *)> CB) const = 0;
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AAIndirectCallInfo"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AAIndirectCallInfo
+ /// This function should return true if the type of the \p AA is
+ /// AADenormalFPMath.
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+/// An abstract Attribute for specializing "dynamic" components of
+/// "denormal-fp-math" and "denormal-fp-math-f32" to a known denormal mode.
+struct AADenormalFPMath
+ : public StateWrapper<DenormalFPMathState, AbstractAttribute> {
+ using Base = StateWrapper<DenormalFPMathState, AbstractAttribute>;
+
+ AADenormalFPMath(const IRPosition &IRP, Attributor &A) : Base(IRP) {}
+
+ /// Create an abstract attribute view for the position \p IRP.
+ static AADenormalFPMath &createForPosition(const IRPosition &IRP,
+ Attributor &A);
+
+ /// See AbstractAttribute::getName()
+ const std::string getName() const override { return "AADenormalFPMath"; }
+
+ /// See AbstractAttribute::getIdAddr()
+ const char *getIdAddr() const override { return &ID; }
+
+ /// This function should return true if the type of the \p AA is
+ /// AADenormalFPMath.
+ static bool classof(const AbstractAttribute *AA) {
+ return (AA->getIdAddr() == &ID);
+ }
+
+ /// Unique ID (due to the unique address)
+ static const char ID;
+};
+
+raw_ostream &operator<<(raw_ostream &, const AAPointerInfo::Access &);
+
+/// Run options, used by the pass manager.
+enum AttributorRunOption {
+ NONE = 0,
+ MODULE = 1 << 0,
+ CGSCC = 1 << 1,
+ ALL = MODULE | CGSCC
+};
+
+namespace AA {
+/// Helper to avoid creating an AA for IR Attributes that might already be set.
+template <Attribute::AttrKind AK, typename AAType = AbstractAttribute>
+bool hasAssumedIRAttr(Attributor &A, const AbstractAttribute *QueryingAA,
+ const IRPosition &IRP, DepClassTy DepClass, bool &IsKnown,
+ bool IgnoreSubsumingPositions = false,
+ const AAType **AAPtr = nullptr) {
+ IsKnown = false;
+ switch (AK) {
+#define CASE(ATTRNAME, AANAME, ...) \
+ case Attribute::ATTRNAME: { \
+ if (AANAME::isImpliedByIR(A, IRP, AK, IgnoreSubsumingPositions)) \
+ return IsKnown = true; \
+ if (!QueryingAA) \
+ return false; \
+ const auto *AA = A.getAAFor<AANAME>(*QueryingAA, IRP, DepClass); \
+ if (AAPtr) \
+ *AAPtr = reinterpret_cast<const AAType *>(AA); \
+ if (!AA || !AA->isAssumed(__VA_ARGS__)) \
+ return false; \
+ IsKnown = AA->isKnown(__VA_ARGS__); \
+ return true; \
+ }
+ CASE(NoUnwind, AANoUnwind, );
+ CASE(WillReturn, AAWillReturn, );
+ CASE(NoFree, AANoFree, );
+ CASE(NoCapture, AANoCapture, );
+ CASE(NoRecurse, AANoRecurse, );
+ CASE(NoReturn, AANoReturn, );
+ CASE(NoSync, AANoSync, );
+ CASE(NoAlias, AANoAlias, );
+ CASE(NonNull, AANonNull, );
+ CASE(MustProgress, AAMustProgress, );
+ CASE(NoUndef, AANoUndef, );
+ CASE(ReadNone, AAMemoryBehavior, AAMemoryBehavior::NO_ACCESSES);
+ CASE(ReadOnly, AAMemoryBehavior, AAMemoryBehavior::NO_WRITES);
+ CASE(WriteOnly, AAMemoryBehavior, AAMemoryBehavior::NO_READS);
+#undef CASE
+ default:
+ llvm_unreachable("hasAssumedIRAttr not available for this attribute kind");
+ };
+}
+} // namespace AA
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index e69de29bb2d1d..24bc6b9ebfc02 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -0,0 +1,13228 @@
+//===- AttributorAttributes.cpp - Attributes for Attributor deduction -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// See the Attributor.h file comment and the class descriptions in that file for
+// more information.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/Attributor.h"
+
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SCCIterator.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetOperations.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AssumeBundleQueries.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/CaptureTracking.h"
+#include "llvm/Analysis/CycleAnalysis.h"
+#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LazyValueInfo.h"
+#include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Argument.h"
+#include "llvm/IR/Assumptions.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InlineAsm.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/IR/IntrinsicsNVPTX.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/NoFolder.h"
+#include "llvm/IR/Value.h"
+#include "llvm/IR/ValueHandle.h"
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/GraphWriter.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/TypeSize.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/CallPromotionUtils.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/ValueMapper.h"
+#include <cassert>
+#include <numeric>
+#include <optional>
+#include <string>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "attributor"
+
+static cl::opt<bool> ManifestInternal(
+ "attributor-manifest-internal", cl::Hidden,
+ cl::desc("Manifest Attributor internal string attributes."),
+ cl::init(false));
+
+static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size", cl::init(128),
+ cl::Hidden);
+
+template <>
+unsigned llvm::PotentialConstantIntValuesState::MaxPotentialValues = 0;
+
+template <> unsigned llvm::PotentialLLVMValuesState::MaxPotentialValues = -1;
+
+static cl::opt<unsigned, true> MaxPotentialValues(
+ "attributor-max-potential-values", cl::Hidden,
+ cl::desc("Maximum number of potential values to be "
+ "tracked for each position."),
+ cl::location(llvm::PotentialConstantIntValuesState::MaxPotentialValues),
+ cl::init(7));
+
+static cl::opt<int> MaxPotentialValuesIterations(
+ "attributor-max-potential-values-iterations", cl::Hidden,
+ cl::desc(
+ "Maximum number of iterations we keep dismantling potential values."),
+ cl::init(64));
+
+STATISTIC(NumAAs, "Number of abstract attributes created");
+
+// Some helper macros to deal with statistics tracking.
+//
+// Usage:
+// For simple IR attribute tracking overload trackStatistics in the abstract
+// attribute and choose the right STATS_DECLTRACK_********* macro,
+// e.g.,:
+// void trackStatistics() const override {
+// STATS_DECLTRACK_ARG_ATTR(returned)
+// }
+// If there is a single "increment" side one can use the macro
+// STATS_DECLTRACK with a custom message. If there are multiple increment
+// sides, STATS_DECL and STATS_TRACK can also be used separately.
+//
+#define BUILD_STAT_MSG_IR_ATTR(TYPE, NAME) \
+ ("Number of " #TYPE " marked '" #NAME "'")
+#define BUILD_STAT_NAME(NAME, TYPE) NumIR##TYPE##_##NAME
+#define STATS_DECL_(NAME, MSG) STATISTIC(NAME, MSG);
+#define STATS_DECL(NAME, TYPE, MSG) \
+ STATS_DECL_(BUILD_STAT_NAME(NAME, TYPE), MSG);
+#define STATS_TRACK(NAME, TYPE) ++(BUILD_STAT_NAME(NAME, TYPE));
+#define STATS_DECLTRACK(NAME, TYPE, MSG) \
+ { \
+ STATS_DECL(NAME, TYPE, MSG) \
+ STATS_TRACK(NAME, TYPE) \
+ }
+#define STATS_DECLTRACK_ARG_ATTR(NAME) \
+ STATS_DECLTRACK(NAME, Arguments, BUILD_STAT_MSG_IR_ATTR(arguments, NAME))
+#define STATS_DECLTRACK_CSARG_ATTR(NAME) \
+ STATS_DECLTRACK(NAME, CSArguments, \
+ BUILD_STAT_MSG_IR_ATTR(call site arguments, NAME))
+#define STATS_DECLTRACK_FN_ATTR(NAME) \
+ STATS_DECLTRACK(NAME, Function, BUILD_STAT_MSG_IR_ATTR(functions, NAME))
+#define STATS_DECLTRACK_CS_ATTR(NAME) \
+ STATS_DECLTRACK(NAME, CS, BUILD_STAT_MSG_IR_ATTR(call site, NAME))
+#define STATS_DECLTRACK_FNRET_ATTR(NAME) \
+ STATS_DECLTRACK(NAME, FunctionReturn, \
+ BUILD_STAT_MSG_IR_ATTR(function returns, NAME))
+#define STATS_DECLTRACK_CSRET_ATTR(NAME) \
+ STATS_DECLTRACK(NAME, CSReturn, \
+ BUILD_STAT_MSG_IR_ATTR(call site returns, NAME))
+#define STATS_DECLTRACK_FLOATING_ATTR(NAME) \
+ STATS_DECLTRACK(NAME, Floating, \
+ ("Number of floating values known to be '" #NAME "'"))
+
+// Specialization of the operator<< for abstract attributes subclasses. This
+// disambiguates situations where multiple operators are applicable.
+namespace llvm {
+#define PIPE_OPERATOR(CLASS) \
+ raw_ostream &operator<<(raw_ostream &OS, const CLASS &AA) { \
+ return OS << static_cast<const AbstractAttribute &>(AA); \
+ }
+
+PIPE_OPERATOR(AAIsDead)
+PIPE_OPERATOR(AANoUnwind)
+PIPE_OPERATOR(AANoSync)
+PIPE_OPERATOR(AANoRecurse)
+PIPE_OPERATOR(AANonConvergent)
+PIPE_OPERATOR(AAWillReturn)
+PIPE_OPERATOR(AANoReturn)
+PIPE_OPERATOR(AANonNull)
+PIPE_OPERATOR(AAMustProgress)
+PIPE_OPERATOR(AANoAlias)
+PIPE_OPERATOR(AADereferenceable)
+PIPE_OPERATOR(AAAlign)
+PIPE_OPERATOR(AAInstanceInfo)
+PIPE_OPERATOR(AANoCapture)
+PIPE_OPERATOR(AAValueSimplify)
+PIPE_OPERATOR(AANoFree)
+PIPE_OPERATOR(AAHeapToStack)
+PIPE_OPERATOR(AAIntraFnReachability)
+PIPE_OPERATOR(AAMemoryBehavior)
+PIPE_OPERATOR(AAMemoryLocation)
+PIPE_OPERATOR(AAValueConstantRange)
+PIPE_OPERATOR(AAPrivatizablePtr)
+PIPE_OPERATOR(AAUndefinedBehavior)
+PIPE_OPERATOR(AAPotentialConstantValues)
+PIPE_OPERATOR(AAPotentialValues)
+PIPE_OPERATOR(AANoUndef)
+PIPE_OPERATOR(AANoFPClass)
+PIPE_OPERATOR(AACallEdges)
+PIPE_OPERATOR(AAInterFnReachability)
+PIPE_OPERATOR(AAPointerInfo)
+PIPE_OPERATOR(AAAssumptionInfo)
+PIPE_OPERATOR(AAUnderlyingObjects)
+PIPE_OPERATOR(AAAddressSpace)
+PIPE_OPERATOR(AAAllocationInfo)
+PIPE_OPERATOR(AAIndirectCallInfo)
+PIPE_OPERATOR(AAGlobalValueInfo)
+PIPE_OPERATOR(AADenormalFPMath)
+
+#undef PIPE_OPERATOR
+
+template <>
+ChangeStatus clampStateAndIndicateChange<DerefState>(DerefState &S,
+ const DerefState &R) {
+ ChangeStatus CS0 =
+ clampStateAndIndicateChange(S.DerefBytesState, R.DerefBytesState);
+ ChangeStatus CS1 = clampStateAndIndicateChange(S.GlobalState, R.GlobalState);
+ return CS0 | CS1;
+}
+
+} // namespace llvm
+
+static bool mayBeInCycle(const CycleInfo *CI, const Instruction *I,
+ bool HeaderOnly, Cycle **CPtr = nullptr) {
+ if (!CI)
+ return true;
+ auto *BB = I->getParent();
+ auto *C = CI->getCycle(BB);
+ if (!C)
+ return false;
+ if (CPtr)
+ *CPtr = C;
+ return !HeaderOnly || BB == C->getHeader();
+}
+
+/// Checks if a type could have padding bytes.
+static bool isDenselyPacked(Type *Ty, const DataLayout &DL) {
+ // There is no size information, so be conservative.
+ if (!Ty->isSized())
+ return false;
+
+ // If the alloc size is not equal to the storage size, then there are padding
+ // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
+ if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
+ return false;
+
+ // FIXME: This isn't the right way to check for padding in vectors with
+ // non-byte-size elements.
+ if (VectorType *SeqTy = dyn_cast<VectorType>(Ty))
+ return isDenselyPacked(SeqTy->getElementType(), DL);
+
+ // For array types, check for padding within members.
+ if (ArrayType *SeqTy = dyn_cast<ArrayType>(Ty))
+ return isDenselyPacked(SeqTy->getElementType(), DL);
+
+ if (!isa<StructType>(Ty))
+ return true;
+
+ // Check for padding within and between elements of a struct.
+ StructType *StructTy = cast<StructType>(Ty);
+ const StructLayout *Layout = DL.getStructLayout(StructTy);
+ uint64_t StartPos = 0;
+ for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
+ Type *ElTy = StructTy->getElementType(I);
+ if (!isDenselyPacked(ElTy, DL))
+ return false;
+ if (StartPos != Layout->getElementOffsetInBits(I))
+ return false;
+ StartPos += DL.getTypeAllocSizeInBits(ElTy);
+ }
+
+ return true;
+}
+
+/// Get pointer operand of memory accessing instruction. If \p I is
+/// not a memory accessing instruction, return nullptr. If \p AllowVolatile,
+/// is set to false and the instruction is volatile, return nullptr.
+static const Value *getPointerOperand(const Instruction *I,
+ bool AllowVolatile) {
+ if (!AllowVolatile && I->isVolatile())
+ return nullptr;
+
+ if (auto *LI = dyn_cast<LoadInst>(I)) {
+ return LI->getPointerOperand();
+ }
+
+ if (auto *SI = dyn_cast<StoreInst>(I)) {
+ return SI->getPointerOperand();
+ }
+
+ if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(I)) {
+ return CXI->getPointerOperand();
+ }
+
+ if (auto *RMWI = dyn_cast<AtomicRMWInst>(I)) {
+ return RMWI->getPointerOperand();
+ }
+
+ return nullptr;
+}
+
+/// Helper function to create a pointer based on \p Ptr, and advanced by \p
+/// Offset bytes.
+static Value *constructPointer(Value *Ptr, int64_t Offset,
+ IRBuilder<NoFolder> &IRB) {
+ LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset
+ << "-bytes\n");
+
+ if (Offset)
+ Ptr = IRB.CreatePtrAdd(Ptr, IRB.getInt64(Offset),
+ Ptr->getName() + ".b" + Twine(Offset));
+ return Ptr;
+}
+
+static const Value *
+stripAndAccumulateOffsets(Attributor &A, const AbstractAttribute &QueryingAA,
+ const Value *Val, const DataLayout &DL, APInt &Offset,
+ bool GetMinOffset, bool AllowNonInbounds,
+ bool UseAssumed = false) {
+
+ auto AttributorAnalysis = [&](Value &V, APInt &ROffset) -> bool {
+ const IRPosition &Pos = IRPosition::value(V);
+ // Only track dependence if we are going to use the assumed info.
+ const AAValueConstantRange *ValueConstantRangeAA =
+ A.getAAFor<AAValueConstantRange>(QueryingAA, Pos,
+ UseAssumed ? DepClassTy::OPTIONAL
+ : DepClassTy::NONE);
+ if (!ValueConstantRangeAA)
+ return false;
+ ConstantRange Range = UseAssumed ? ValueConstantRangeAA->getAssumed()
+ : ValueConstantRangeAA->getKnown();
+ if (Range.isFullSet())
+ return false;
+
+ // We can only use the lower part of the range because the upper part can
+ // be higher than what the value can really be.
+ if (GetMinOffset)
+ ROffset = Range.getSignedMin();
+ else
+ ROffset = Range.getSignedMax();
+ return true;
+ };
+
+ return Val->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds,
+ /* AllowInvariant */ true,
+ AttributorAnalysis);
+}
+
+static const Value *
+getMinimalBaseOfPointer(Attributor &A, const AbstractAttribute &QueryingAA,
+ const Value *Ptr, int64_t &BytesOffset,
+ const DataLayout &DL, bool AllowNonInbounds = false) {
+ APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
+ const Value *Base =
+ stripAndAccumulateOffsets(A, QueryingAA, Ptr, DL, OffsetAPInt,
+ /* GetMinOffset */ true, AllowNonInbounds);
+
+ BytesOffset = OffsetAPInt.getSExtValue();
+ return Base;
+}
+
+/// Clamp the information known for all returned values of a function
+/// (identified by \p QueryingAA) into \p S.
+template <typename AAType, typename StateType = typename AAType::StateType,
+ Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
+ bool RecurseForSelectAndPHI = true>
+static void clampReturnedValueStates(
+ Attributor &A, const AAType &QueryingAA, StateType &S,
+ const IRPosition::CallBaseContext *CBContext = nullptr) {
+ LLVM_DEBUG(dbgs() << "[Attributor] Clamp return value states for "
+ << QueryingAA << " into " << S << "\n");
+
+ assert((QueryingAA.getIRPosition().getPositionKind() ==
+ IRPosition::IRP_RETURNED ||
+ QueryingAA.getIRPosition().getPositionKind() ==
+ IRPosition::IRP_CALL_SITE_RETURNED) &&
+ "Can only clamp returned value states for a function returned or call "
+ "site returned position!");
+
+ // Use an optional state as there might not be any return values and we want
+ // to join (IntegerState::operator&) the state of all there are.
+ std::optional<StateType> T;
+
+ // Callback for each possibly returned value.
+ auto CheckReturnValue = [&](Value &RV) -> bool {
+ const IRPosition &RVPos = IRPosition::value(RV, CBContext);
+ // If possible, use the hasAssumedIRAttr interface.
+ if (Attribute::isEnumAttrKind(IRAttributeKind)) {
+ bool IsKnown;
+ return AA::hasAssumedIRAttr<IRAttributeKind>(
+ A, &QueryingAA, RVPos, DepClassTy::REQUIRED, IsKnown);
+ }
+
+ const AAType *AA =
+ A.getAAFor<AAType>(QueryingAA, RVPos, DepClassTy::REQUIRED);
+ if (!AA)
+ return false;
+ LLVM_DEBUG(dbgs() << "[Attributor] RV: " << RV
+ << " AA: " << AA->getAsStr(&A) << " @ " << RVPos << "\n");
+ const StateType &AAS = AA->getState();
+ if (!T)
+ T = StateType::getBestState(AAS);
+ *T &= AAS;
+ LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " RV State: " << T
+ << "\n");
+ return T->isValidState();
+ };
+
+ if (!A.checkForAllReturnedValues(CheckReturnValue, QueryingAA,
+ AA::ValueScope::Intraprocedural,
+ RecurseForSelectAndPHI))
+ S.indicatePessimisticFixpoint();
+ else if (T)
+ S ^= *T;
+}
+
+namespace {
+/// Helper class for generic deduction: return value -> returned position.
+template <typename AAType, typename BaseType,
+ typename StateType = typename BaseType::StateType,
+ bool PropagateCallBaseContext = false,
+ Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind,
+ bool RecurseForSelectAndPHI = true>
+struct AAReturnedFromReturnedValues : public BaseType {
+ AAReturnedFromReturnedValues(const IRPosition &IRP, Attributor &A)
+ : BaseType(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ StateType S(StateType::getBestState(this->getState()));
+ clampReturnedValueStates<AAType, StateType, IRAttributeKind, RecurseForSelectAndPHI>(
+ A, *this, S,
+ PropagateCallBaseContext ? this->getCallBaseContext() : nullptr);
+ // TODO: If we know we visited all returned values, thus no are assumed
+ // dead, we can take the known information from the state T.
+ return clampStateAndIndicateChange<StateType>(this->getState(), S);
+ }
+};
+
+/// Clamp the information known at all call sites for a given argument
+/// (identified by \p QueryingAA) into \p S.
+template <typename AAType, typename StateType = typename AAType::StateType,
+ Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
+static void clampCallSiteArgumentStates(Attributor &A, const AAType &QueryingAA,
+ StateType &S) {
+ LLVM_DEBUG(dbgs() << "[Attributor] Clamp call site argument states for "
+ << QueryingAA << " into " << S << "\n");
+
+ assert(QueryingAA.getIRPosition().getPositionKind() ==
+ IRPosition::IRP_ARGUMENT &&
+ "Can only clamp call site argument states for an argument position!");
+
+ // Use an optional state as there might not be any return values and we want
+ // to join (IntegerState::operator&) the state of all there are.
+ std::optional<StateType> T;
+
+ // The argument number which is also the call site argument number.
+ unsigned ArgNo = QueryingAA.getIRPosition().getCallSiteArgNo();
+
+ auto CallSiteCheck = [&](AbstractCallSite ACS) {
+ const IRPosition &ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
+ // Check if a coresponding argument was found or if it is on not associated
+ // (which can happen for callback calls).
+ if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
+ return false;
+
+ // If possible, use the hasAssumedIRAttr interface.
+ if (Attribute::isEnumAttrKind(IRAttributeKind)) {
+ bool IsKnown;
+ return AA::hasAssumedIRAttr<IRAttributeKind>(
+ A, &QueryingAA, ACSArgPos, DepClassTy::REQUIRED, IsKnown);
+ }
+
+ const AAType *AA =
+ A.getAAFor<AAType>(QueryingAA, ACSArgPos, DepClassTy::REQUIRED);
+ if (!AA)
+ return false;
+ LLVM_DEBUG(dbgs() << "[Attributor] ACS: " << *ACS.getInstruction()
+ << " AA: " << AA->getAsStr(&A) << " @" << ACSArgPos
+ << "\n");
+ const StateType &AAS = AA->getState();
+ if (!T)
+ T = StateType::getBestState(AAS);
+ *T &= AAS;
+ LLVM_DEBUG(dbgs() << "[Attributor] AA State: " << AAS << " CSA State: " << T
+ << "\n");
+ return T->isValidState();
+ };
+
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllCallSites(CallSiteCheck, QueryingAA, true,
+ UsedAssumedInformation))
+ S.indicatePessimisticFixpoint();
+ else if (T)
+ S ^= *T;
+}
+
+/// This function is the bridge between argument position and the call base
+/// context.
+template <typename AAType, typename BaseType,
+ typename StateType = typename AAType::StateType,
+ Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
+bool getArgumentStateFromCallBaseContext(Attributor &A,
+ BaseType &QueryingAttribute,
+ IRPosition &Pos, StateType &State) {
+ assert((Pos.getPositionKind() == IRPosition::IRP_ARGUMENT) &&
+ "Expected an 'argument' position !");
+ const CallBase *CBContext = Pos.getCallBaseContext();
+ if (!CBContext)
+ return false;
+
+ int ArgNo = Pos.getCallSiteArgNo();
+ assert(ArgNo >= 0 && "Invalid Arg No!");
+ const IRPosition CBArgPos = IRPosition::callsite_argument(*CBContext, ArgNo);
+
+ // If possible, use the hasAssumedIRAttr interface.
+ if (Attribute::isEnumAttrKind(IRAttributeKind)) {
+ bool IsKnown;
+ return AA::hasAssumedIRAttr<IRAttributeKind>(
+ A, &QueryingAttribute, CBArgPos, DepClassTy::REQUIRED, IsKnown);
+ }
+
+ const auto *AA =
+ A.getAAFor<AAType>(QueryingAttribute, CBArgPos, DepClassTy::REQUIRED);
+ if (!AA)
+ return false;
+ const StateType &CBArgumentState =
+ static_cast<const StateType &>(AA->getState());
+
+ LLVM_DEBUG(dbgs() << "[Attributor] Briding Call site context to argument"
+ << "Position:" << Pos << "CB Arg state:" << CBArgumentState
+ << "\n");
+
+ // NOTE: If we want to do call site grouping it should happen here.
+ State ^= CBArgumentState;
+ return true;
+}
+
+/// Helper class for generic deduction: call site argument -> argument position.
+template <typename AAType, typename BaseType,
+ typename StateType = typename AAType::StateType,
+ bool BridgeCallBaseContext = false,
+ Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
+struct AAArgumentFromCallSiteArguments : public BaseType {
+ AAArgumentFromCallSiteArguments(const IRPosition &IRP, Attributor &A)
+ : BaseType(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ StateType S = StateType::getBestState(this->getState());
+
+ if (BridgeCallBaseContext) {
+ bool Success =
+ getArgumentStateFromCallBaseContext<AAType, BaseType, StateType,
+ IRAttributeKind>(
+ A, *this, this->getIRPosition(), S);
+ if (Success)
+ return clampStateAndIndicateChange<StateType>(this->getState(), S);
+ }
+ clampCallSiteArgumentStates<AAType, StateType, IRAttributeKind>(A, *this,
+ S);
+
+ // TODO: If we know we visited all incoming values, thus no are assumed
+ // dead, we can take the known information from the state T.
+ return clampStateAndIndicateChange<StateType>(this->getState(), S);
+ }
+};
+
+/// Helper class for generic replication: function returned -> cs returned.
+template <typename AAType, typename BaseType,
+ typename StateType = typename BaseType::StateType,
+ bool IntroduceCallBaseContext = false,
+ Attribute::AttrKind IRAttributeKind = AAType::IRAttributeKind>
+struct AACalleeToCallSite : public BaseType {
+ AACalleeToCallSite(const IRPosition &IRP, Attributor &A) : BaseType(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto IRPKind = this->getIRPosition().getPositionKind();
+ assert((IRPKind == IRPosition::IRP_CALL_SITE_RETURNED ||
+ IRPKind == IRPosition::IRP_CALL_SITE) &&
+ "Can only wrap function returned positions for call site "
+ "returned positions!");
+ auto &S = this->getState();
+
+ CallBase &CB = cast<CallBase>(this->getAnchorValue());
+ if (IntroduceCallBaseContext)
+ LLVM_DEBUG(dbgs() << "[Attributor] Introducing call base context:" << CB
+ << "\n");
+
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ auto CalleePred = [&](ArrayRef<const Function *> Callees) {
+ for (const Function *Callee : Callees) {
+ IRPosition FnPos =
+ IRPKind == llvm::IRPosition::IRP_CALL_SITE_RETURNED
+ ? IRPosition::returned(*Callee,
+ IntroduceCallBaseContext ? &CB : nullptr)
+ : IRPosition::function(
+ *Callee, IntroduceCallBaseContext ? &CB : nullptr);
+ // If possible, use the hasAssumedIRAttr interface.
+ if (Attribute::isEnumAttrKind(IRAttributeKind)) {
+ bool IsKnown;
+ if (!AA::hasAssumedIRAttr<IRAttributeKind>(
+ A, this, FnPos, DepClassTy::REQUIRED, IsKnown))
+ return false;
+ continue;
+ }
+
+ const AAType *AA =
+ A.getAAFor<AAType>(*this, FnPos, DepClassTy::REQUIRED);
+ if (!AA)
+ return false;
+ Changed |= clampStateAndIndicateChange(S, AA->getState());
+ if (S.isAtFixpoint())
+ return S.isValidState();
+ }
+ return true;
+ };
+ if (!A.checkForAllCallees(CalleePred, *this, CB))
+ return S.indicatePessimisticFixpoint();
+ return Changed;
+ }
+};
+
+/// Helper function to accumulate uses.
+template <class AAType, typename StateType = typename AAType::StateType>
+static void followUsesInContext(AAType &AA, Attributor &A,
+ MustBeExecutedContextExplorer &Explorer,
+ const Instruction *CtxI,
+ SetVector<const Use *> &Uses,
+ StateType &State) {
+ auto EIt = Explorer.begin(CtxI), EEnd = Explorer.end(CtxI);
+ for (unsigned u = 0; u < Uses.size(); ++u) {
+ const Use *U = Uses[u];
+ if (const Instruction *UserI = dyn_cast<Instruction>(U->getUser())) {
+ bool Found = Explorer.findInContextOf(UserI, EIt, EEnd);
+ if (Found && AA.followUseInMBEC(A, U, UserI, State))
+ for (const Use &Us : UserI->uses())
+ Uses.insert(&Us);
+ }
+ }
+}
+
+/// Use the must-be-executed-context around \p I to add information into \p S.
+/// The AAType class is required to have `followUseInMBEC` method with the
+/// following signature and behaviour:
+///
+/// bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I)
+/// U - Underlying use.
+/// I - The user of the \p U.
+/// Returns true if the value should be tracked transitively.
+///
+template <class AAType, typename StateType = typename AAType::StateType>
+static void followUsesInMBEC(AAType &AA, Attributor &A, StateType &S,
+ Instruction &CtxI) {
+ MustBeExecutedContextExplorer *Explorer =
+ A.getInfoCache().getMustBeExecutedContextExplorer();
+ if (!Explorer)
+ return;
+
+ // Container for (transitive) uses of the associated value.
+ SetVector<const Use *> Uses;
+ for (const Use &U : AA.getIRPosition().getAssociatedValue().uses())
+ Uses.insert(&U);
+
+ followUsesInContext<AAType>(AA, A, *Explorer, &CtxI, Uses, S);
+
+ if (S.isAtFixpoint())
+ return;
+
+ SmallVector<const BranchInst *, 4> BrInsts;
+ auto Pred = [&](const Instruction *I) {
+ if (const BranchInst *Br = dyn_cast<BranchInst>(I))
+ if (Br->isConditional())
+ BrInsts.push_back(Br);
+ return true;
+ };
+
+ // Here, accumulate conditional branch instructions in the context. We
+ // explore the child paths and collect the known states. The disjunction of
+ // those states can be merged to its own state. Let ParentState_i be a state
+ // to indicate the known information for an i-th branch instruction in the
+ // context. ChildStates are created for its successors respectively.
+ //
+ // ParentS_1 = ChildS_{1, 1} /\ ChildS_{1, 2} /\ ... /\ ChildS_{1, n_1}
+ // ParentS_2 = ChildS_{2, 1} /\ ChildS_{2, 2} /\ ... /\ ChildS_{2, n_2}
+ // ...
+ // ParentS_m = ChildS_{m, 1} /\ ChildS_{m, 2} /\ ... /\ ChildS_{m, n_m}
+ //
+ // Known State |= ParentS_1 \/ ParentS_2 \/... \/ ParentS_m
+ //
+ // FIXME: Currently, recursive branches are not handled. For example, we
+ // can't deduce that ptr must be dereferenced in below function.
+ //
+ // void f(int a, int c, int *ptr) {
+ // if(a)
+ // if (b) {
+ // *ptr = 0;
+ // } else {
+ // *ptr = 1;
+ // }
+ // else {
+ // if (b) {
+ // *ptr = 0;
+ // } else {
+ // *ptr = 1;
+ // }
+ // }
+ // }
+
+ Explorer->checkForAllContext(&CtxI, Pred);
+ for (const BranchInst *Br : BrInsts) {
+ StateType ParentState;
+
+ // The known state of the parent state is a conjunction of children's
+ // known states so it is initialized with a best state.
+ ParentState.indicateOptimisticFixpoint();
+
+ for (const BasicBlock *BB : Br->successors()) {
+ StateType ChildState;
+
+ size_t BeforeSize = Uses.size();
+ followUsesInContext(AA, A, *Explorer, &BB->front(), Uses, ChildState);
+
+ // Erase uses which only appear in the child.
+ for (auto It = Uses.begin() + BeforeSize; It != Uses.end();)
+ It = Uses.erase(It);
+
+ ParentState &= ChildState;
+ }
+
+ // Use only known state.
+ S += ParentState;
+ }
+}
+} // namespace
+
+/// ------------------------ PointerInfo ---------------------------------------
+
+namespace llvm {
+namespace AA {
+namespace PointerInfo {
+
+struct State;
+
+} // namespace PointerInfo
+} // namespace AA
+
+/// Helper for AA::PointerInfo::Access DenseMap/Set usage.
+template <>
+struct DenseMapInfo<AAPointerInfo::Access> : DenseMapInfo<Instruction *> {
+ using Access = AAPointerInfo::Access;
+ static inline Access getEmptyKey();
+ static inline Access getTombstoneKey();
+ static unsigned getHashValue(const Access &A);
+ static bool isEqual(const Access &LHS, const Access &RHS);
+};
+
+/// Helper that allows RangeTy as a key in a DenseMap.
+template <> struct DenseMapInfo<AA::RangeTy> {
+ static inline AA::RangeTy getEmptyKey() {
+ auto EmptyKey = DenseMapInfo<int64_t>::getEmptyKey();
+ return AA::RangeTy{EmptyKey, EmptyKey};
+ }
+
+ static inline AA::RangeTy getTombstoneKey() {
+ auto TombstoneKey = DenseMapInfo<int64_t>::getTombstoneKey();
+ return AA::RangeTy{TombstoneKey, TombstoneKey};
+ }
+
+ static unsigned getHashValue(const AA::RangeTy &Range) {
+ return detail::combineHashValue(
+ DenseMapInfo<int64_t>::getHashValue(Range.Offset),
+ DenseMapInfo<int64_t>::getHashValue(Range.Size));
+ }
+
+ static bool isEqual(const AA::RangeTy &A, const AA::RangeTy B) {
+ return A == B;
+ }
+};
+
+/// Helper for AA::PointerInfo::Access DenseMap/Set usage ignoring everythign
+/// but the instruction
+struct AccessAsInstructionInfo : DenseMapInfo<Instruction *> {
+ using Base = DenseMapInfo<Instruction *>;
+ using Access = AAPointerInfo::Access;
+ static inline Access getEmptyKey();
+ static inline Access getTombstoneKey();
+ static unsigned getHashValue(const Access &A);
+ static bool isEqual(const Access &LHS, const Access &RHS);
+};
+
+} // namespace llvm
+
+/// A type to track pointer/struct usage and accesses for AAPointerInfo.
+struct AA::PointerInfo::State : public AbstractState {
+ /// Return the best possible representable state.
+ static State getBestState(const State &SIS) { return State(); }
+
+ /// Return the worst possible representable state.
+ static State getWorstState(const State &SIS) {
+ State R;
+ R.indicatePessimisticFixpoint();
+ return R;
+ }
+
+ State() = default;
+ State(State &&SIS) = default;
+
+ const State &getAssumed() const { return *this; }
+
+ /// See AbstractState::isValidState().
+ bool isValidState() const override { return BS.isValidState(); }
+
+ /// See AbstractState::isAtFixpoint().
+ bool isAtFixpoint() const override { return BS.isAtFixpoint(); }
+
+ /// See AbstractState::indicateOptimisticFixpoint().
+ ChangeStatus indicateOptimisticFixpoint() override {
+ BS.indicateOptimisticFixpoint();
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractState::indicatePessimisticFixpoint().
+ ChangeStatus indicatePessimisticFixpoint() override {
+ BS.indicatePessimisticFixpoint();
+ return ChangeStatus::CHANGED;
+ }
+
+ State &operator=(const State &R) {
+ if (this == &R)
+ return *this;
+ BS = R.BS;
+ AccessList = R.AccessList;
+ OffsetBins = R.OffsetBins;
+ RemoteIMap = R.RemoteIMap;
+ return *this;
+ }
+
+ State &operator=(State &&R) {
+ if (this == &R)
+ return *this;
+ std::swap(BS, R.BS);
+ std::swap(AccessList, R.AccessList);
+ std::swap(OffsetBins, R.OffsetBins);
+ std::swap(RemoteIMap, R.RemoteIMap);
+ return *this;
+ }
+
+ /// Add a new Access to the state at offset \p Offset and with size \p Size.
+ /// The access is associated with \p I, writes \p Content (if anything), and
+ /// is of kind \p Kind. If an Access already exists for the same \p I and same
+ /// \p RemoteI, the two are combined, potentially losing information about
+ /// offset and size. The resulting access must now be moved from its original
+ /// OffsetBin to the bin for its new offset.
+ ///
+ /// \Returns CHANGED, if the state changed, UNCHANGED otherwise.
+ ChangeStatus addAccess(Attributor &A, const AAPointerInfo::RangeList &Ranges,
+ Instruction &I, std::optional<Value *> Content,
+ AAPointerInfo::AccessKind Kind, Type *Ty,
+ Instruction *RemoteI = nullptr);
+
+ AAPointerInfo::const_bin_iterator begin() const { return OffsetBins.begin(); }
+ AAPointerInfo::const_bin_iterator end() const { return OffsetBins.end(); }
+ int64_t numOffsetBins() const { return OffsetBins.size(); }
+
+ const AAPointerInfo::Access &getAccess(unsigned Index) const {
+ return AccessList[Index];
+ }
+
+protected:
+ // Every memory instruction results in an Access object. We maintain a list of
+ // all Access objects that we own, along with the following maps:
+ //
+ // - OffsetBins: RangeTy -> { Access }
+ // - RemoteIMap: RemoteI x LocalI -> Access
+ //
+ // A RemoteI is any instruction that accesses memory. RemoteI is different
+ // from LocalI if and only if LocalI is a call; then RemoteI is some
+ // instruction in the callgraph starting from LocalI. Multiple paths in the
+ // callgraph from LocalI to RemoteI may produce multiple accesses, but these
+ // are all combined into a single Access object. This may result in loss of
+ // information in RangeTy in the Access object.
+ SmallVector<AAPointerInfo::Access> AccessList;
+ AAPointerInfo::OffsetBinsTy OffsetBins;
+ DenseMap<const Instruction *, SmallVector<unsigned>> RemoteIMap;
+
+ /// See AAPointerInfo::forallInterferingAccesses.
+ bool forallInterferingAccesses(
+ AA::RangeTy Range,
+ function_ref<bool(const AAPointerInfo::Access &, bool)> CB) const {
+ if (!isValidState())
+ return false;
+
+ for (const auto &It : OffsetBins) {
+ AA::RangeTy ItRange = It.getFirst();
+ if (!Range.mayOverlap(ItRange))
+ continue;
+ bool IsExact = Range == ItRange && !Range.offsetOrSizeAreUnknown();
+ for (auto Index : It.getSecond()) {
+ auto &Access = AccessList[Index];
+ if (!CB(Access, IsExact))
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /// See AAPointerInfo::forallInterferingAccesses.
+ bool forallInterferingAccesses(
+ Instruction &I,
+ function_ref<bool(const AAPointerInfo::Access &, bool)> CB,
+ AA::RangeTy &Range) const {
+ if (!isValidState())
+ return false;
+
+ auto LocalList = RemoteIMap.find(&I);
+ if (LocalList == RemoteIMap.end()) {
+ return true;
+ }
+
+ for (unsigned Index : LocalList->getSecond()) {
+ for (auto &R : AccessList[Index]) {
+ Range &= R;
+ if (Range.offsetAndSizeAreUnknown())
+ break;
+ }
+ }
+ return forallInterferingAccesses(Range, CB);
+ }
+
+private:
+ /// State to track fixpoint and validity.
+ BooleanState BS;
+};
+
+ChangeStatus AA::PointerInfo::State::addAccess(
+ Attributor &A, const AAPointerInfo::RangeList &Ranges, Instruction &I,
+ std::optional<Value *> Content, AAPointerInfo::AccessKind Kind, Type *Ty,
+ Instruction *RemoteI) {
+ RemoteI = RemoteI ? RemoteI : &I;
+
+ // Check if we have an access for this instruction, if not, simply add it.
+ auto &LocalList = RemoteIMap[RemoteI];
+ bool AccExists = false;
+ unsigned AccIndex = AccessList.size();
+ for (auto Index : LocalList) {
+ auto &A = AccessList[Index];
+ if (A.getLocalInst() == &I) {
+ AccExists = true;
+ AccIndex = Index;
+ break;
+ }
+ }
+
+ auto AddToBins = [&](const AAPointerInfo::RangeList &ToAdd) {
+ LLVM_DEBUG(if (ToAdd.size()) dbgs()
+ << "[AAPointerInfo] Inserting access in new offset bins\n";);
+
+ for (auto Key : ToAdd) {
+ LLVM_DEBUG(dbgs() << " key " << Key << "\n");
+ OffsetBins[Key].insert(AccIndex);
+ }
+ };
+
+ if (!AccExists) {
+ AccessList.emplace_back(&I, RemoteI, Ranges, Content, Kind, Ty);
+ assert((AccessList.size() == AccIndex + 1) &&
+ "New Access should have been at AccIndex");
+ LocalList.push_back(AccIndex);
+ AddToBins(AccessList[AccIndex].getRanges());
+ return ChangeStatus::CHANGED;
+ }
+
+ // Combine the new Access with the existing Access, and then update the
+ // mapping in the offset bins.
+ AAPointerInfo::Access Acc(&I, RemoteI, Ranges, Content, Kind, Ty);
+ auto &Current = AccessList[AccIndex];
+ auto Before = Current;
+ Current &= Acc;
+ if (Current == Before)
+ return ChangeStatus::UNCHANGED;
+
+ auto &ExistingRanges = Before.getRanges();
+ auto &NewRanges = Current.getRanges();
+
+ // Ranges that are in the old access but not the new access need to be removed
+ // from the offset bins.
+ AAPointerInfo::RangeList ToRemove;
+ AAPointerInfo::RangeList::set_difference(ExistingRanges, NewRanges, ToRemove);
+ LLVM_DEBUG(if (ToRemove.size()) dbgs()
+ << "[AAPointerInfo] Removing access from old offset bins\n";);
+
+ for (auto Key : ToRemove) {
+ LLVM_DEBUG(dbgs() << " key " << Key << "\n");
+ assert(OffsetBins.count(Key) && "Existing Access must be in some bin.");
+ auto &Bin = OffsetBins[Key];
+ assert(Bin.count(AccIndex) &&
+ "Expected bin to actually contain the Access.");
+ Bin.erase(AccIndex);
+ }
+
+ // Ranges that are in the new access but not the old access need to be added
+ // to the offset bins.
+ AAPointerInfo::RangeList ToAdd;
+ AAPointerInfo::RangeList::set_difference(NewRanges, ExistingRanges, ToAdd);
+ AddToBins(ToAdd);
+ return ChangeStatus::CHANGED;
+}
+
+namespace {
+
+/// A helper containing a list of offsets computed for a Use. Ideally this
+/// list should be strictly ascending, but we ensure that only when we
+/// actually translate the list of offsets to a RangeList.
+struct OffsetInfo {
+ using VecTy = SmallVector<int64_t>;
+ using const_iterator = VecTy::const_iterator;
+ VecTy Offsets;
+
+ const_iterator begin() const { return Offsets.begin(); }
+ const_iterator end() const { return Offsets.end(); }
+
+ bool operator==(const OffsetInfo &RHS) const {
+ return Offsets == RHS.Offsets;
+ }
+
+ bool operator!=(const OffsetInfo &RHS) const { return !(*this == RHS); }
+
+ void insert(int64_t Offset) { Offsets.push_back(Offset); }
+ bool isUnassigned() const { return Offsets.size() == 0; }
+
+ bool isUnknown() const {
+ if (isUnassigned())
+ return false;
+ if (Offsets.size() == 1)
+ return Offsets.front() == AA::RangeTy::Unknown;
+ return false;
+ }
+
+ void setUnknown() {
+ Offsets.clear();
+ Offsets.push_back(AA::RangeTy::Unknown);
+ }
+
+ void addToAll(int64_t Inc) {
+ for (auto &Offset : Offsets) {
+ Offset += Inc;
+ }
+ }
+
+ /// Copy offsets from \p R into the current list.
+ ///
+ /// Ideally all lists should be strictly ascending, but we defer that to the
+ /// actual use of the list. So we just blindly append here.
+ void merge(const OffsetInfo &R) { Offsets.append(R.Offsets); }
+};
+
+#ifndef NDEBUG
+static raw_ostream &operator<<(raw_ostream &OS, const OffsetInfo &OI) {
+ ListSeparator LS;
+ OS << "[";
+ for (auto Offset : OI) {
+ OS << LS << Offset;
+ }
+ OS << "]";
+ return OS;
+}
+#endif // NDEBUG
+
+struct AAPointerInfoImpl
+ : public StateWrapper<AA::PointerInfo::State, AAPointerInfo> {
+ using BaseTy = StateWrapper<AA::PointerInfo::State, AAPointerInfo>;
+ AAPointerInfoImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return std::string("PointerInfo ") +
+ (isValidState() ? (std::string("#") +
+ std::to_string(OffsetBins.size()) + " bins")
+ : "<invalid>");
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ return AAPointerInfo::manifest(A);
+ }
+
+ virtual const_bin_iterator begin() const override { return State::begin(); }
+ virtual const_bin_iterator end() const override { return State::end(); }
+ virtual int64_t numOffsetBins() const override {
+ return State::numOffsetBins();
+ }
+
+ virtual const Access &getBinAccess(unsigned Index) const override {
+ return getAccess(Index);
+ }
+
+ bool forallInterferingAccesses(
+ AA::RangeTy Range,
+ function_ref<bool(const AAPointerInfo::Access &, bool)> CB)
+ const override {
+ return State::forallInterferingAccesses(Range, CB);
+ }
+
+ bool forallInterferingAccesses(
+ Attributor &A, const AbstractAttribute &QueryingAA, Instruction &I,
+ bool FindInterferingWrites, bool FindInterferingReads,
+ function_ref<bool(const Access &, bool)> UserCB, bool &HasBeenWrittenTo,
+ AA::RangeTy &Range,
+ function_ref<bool(const Access &)> SkipCB) const override {
+ HasBeenWrittenTo = false;
+
+ SmallPtrSet<const Access *, 8> DominatingWrites;
+ SmallVector<std::pair<const Access *, bool>, 8> InterferingAccesses;
+
+ Function &Scope = *I.getFunction();
+ bool IsKnownNoSync;
+ bool IsAssumedNoSync = AA::hasAssumedIRAttr<Attribute::NoSync>(
+ A, &QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL,
+ IsKnownNoSync);
+ const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
+ IRPosition::function(Scope), &QueryingAA, DepClassTy::NONE);
+ bool AllInSameNoSyncFn = IsAssumedNoSync;
+ bool InstIsExecutedByInitialThreadOnly =
+ ExecDomainAA && ExecDomainAA->isExecutedByInitialThreadOnly(I);
+
+ // If the function is not ending in aligned barriers, we need the stores to
+ // be in aligned barriers. The load being in one is not sufficient since the
+ // store might be executed by a thread that disappears after, causing the
+ // aligned barrier guarding the load to unblock and the load to read a value
+ // that has no CFG path to the load.
+ bool InstIsExecutedInAlignedRegion =
+ FindInterferingReads && ExecDomainAA &&
+ ExecDomainAA->isExecutedInAlignedRegion(A, I);
+
+ if (InstIsExecutedInAlignedRegion || InstIsExecutedByInitialThreadOnly)
+ A.recordDependence(*ExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
+
+ InformationCache &InfoCache = A.getInfoCache();
+ bool IsThreadLocalObj =
+ AA::isAssumedThreadLocalObject(A, getAssociatedValue(), *this);
+
+ // Helper to determine if we need to consider threading, which we cannot
+ // right now. However, if the function is (assumed) nosync or the thread
+ // executing all instructions is the main thread only we can ignore
+ // threading. Also, thread-local objects do not require threading reasoning.
+ // Finally, we can ignore threading if either access is executed in an
+ // aligned region.
+ auto CanIgnoreThreadingForInst = [&](const Instruction &I) -> bool {
+ if (IsThreadLocalObj || AllInSameNoSyncFn)
+ return true;
+ const auto *FnExecDomainAA =
+ I.getFunction() == &Scope
+ ? ExecDomainAA
+ : A.lookupAAFor<AAExecutionDomain>(
+ IRPosition::function(*I.getFunction()), &QueryingAA,
+ DepClassTy::NONE);
+ if (!FnExecDomainAA)
+ return false;
+ if (InstIsExecutedInAlignedRegion ||
+ (FindInterferingWrites &&
+ FnExecDomainAA->isExecutedInAlignedRegion(A, I))) {
+ A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
+ return true;
+ }
+ if (InstIsExecutedByInitialThreadOnly &&
+ FnExecDomainAA->isExecutedByInitialThreadOnly(I)) {
+ A.recordDependence(*FnExecDomainAA, QueryingAA, DepClassTy::OPTIONAL);
+ return true;
+ }
+ return false;
+ };
+
+ // Helper to determine if the access is executed by the same thread as the
+ // given instruction, for now it is sufficient to avoid any potential
+ // threading effects as we cannot deal with them anyway.
+ auto CanIgnoreThreading = [&](const Access &Acc) -> bool {
+ return CanIgnoreThreadingForInst(*Acc.getRemoteInst()) ||
+ (Acc.getRemoteInst() != Acc.getLocalInst() &&
+ CanIgnoreThreadingForInst(*Acc.getLocalInst()));
+ };
+
+ // TODO: Use inter-procedural reachability and dominance.
+ bool IsKnownNoRecurse;
+ AA::hasAssumedIRAttr<Attribute::NoRecurse>(
+ A, this, IRPosition::function(Scope), DepClassTy::OPTIONAL,
+ IsKnownNoRecurse);
+
+ // TODO: Use reaching kernels from AAKernelInfo (or move it to
+ // AAExecutionDomain) such that we allow scopes other than kernels as long
+ // as the reaching kernels are disjoint.
+ bool InstInKernel = Scope.hasFnAttribute("kernel");
+ bool ObjHasKernelLifetime = false;
+ const bool UseDominanceReasoning =
+ FindInterferingWrites && IsKnownNoRecurse;
+ const DominatorTree *DT =
+ InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(Scope);
+
+ // Helper to check if a value has "kernel lifetime", that is it will not
+ // outlive a GPU kernel. This is true for shared, constant, and local
+ // globals on AMD and NVIDIA GPUs.
+ auto HasKernelLifetime = [&](Value *V, Module &M) {
+ if (!AA::isGPU(M))
+ return false;
+ switch (AA::GPUAddressSpace(V->getType()->getPointerAddressSpace())) {
+ case AA::GPUAddressSpace::Shared:
+ case AA::GPUAddressSpace::Constant:
+ case AA::GPUAddressSpace::Local:
+ return true;
+ default:
+ return false;
+ };
+ };
+
+ // The IsLiveInCalleeCB will be used by the AA::isPotentiallyReachable query
+ // to determine if we should look at reachability from the callee. For
+ // certain pointers we know the lifetime and we do not have to step into the
+ // callee to determine reachability as the pointer would be dead in the
+ // callee. See the conditional initialization below.
+ std::function<bool(const Function &)> IsLiveInCalleeCB;
+
+ if (auto *AI = dyn_cast<AllocaInst>(&getAssociatedValue())) {
+ // If the alloca containing function is not recursive the alloca
+ // must be dead in the callee.
+ const Function *AIFn = AI->getFunction();
+ ObjHasKernelLifetime = AIFn->hasFnAttribute("kernel");
+ bool IsKnownNoRecurse;
+ if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
+ A, this, IRPosition::function(*AIFn), DepClassTy::OPTIONAL,
+ IsKnownNoRecurse)) {
+ IsLiveInCalleeCB = [AIFn](const Function &Fn) { return AIFn != &Fn; };
+ }
+ } else if (auto *GV = dyn_cast<GlobalValue>(&getAssociatedValue())) {
+ // If the global has kernel lifetime we can stop if we reach a kernel
+ // as it is "dead" in the (unknown) callees.
+ ObjHasKernelLifetime = HasKernelLifetime(GV, *GV->getParent());
+ if (ObjHasKernelLifetime)
+ IsLiveInCalleeCB = [](const Function &Fn) {
+ return !Fn.hasFnAttribute("kernel");
+ };
+ }
+
+ // Set of accesses/instructions that will overwrite the result and are
+ // therefore blockers in the reachability traversal.
+ AA::InstExclusionSetTy ExclusionSet;
+
+ auto AccessCB = [&](const Access &Acc, bool Exact) {
+ Function *AccScope = Acc.getRemoteInst()->getFunction();
+ bool AccInSameScope = AccScope == &Scope;
+
+ // If the object has kernel lifetime we can ignore accesses only reachable
+ // by other kernels. For now we only skip accesses *in* other kernels.
+ if (InstInKernel && ObjHasKernelLifetime && !AccInSameScope &&
+ AccScope->hasFnAttribute("kernel"))
+ return true;
+
+ if (Exact && Acc.isMustAccess() && Acc.getRemoteInst() != &I) {
+ if (Acc.isWrite() || (isa<LoadInst>(I) && Acc.isWriteOrAssumption()))
+ ExclusionSet.insert(Acc.getRemoteInst());
+ }
+
+ if ((!FindInterferingWrites || !Acc.isWriteOrAssumption()) &&
+ (!FindInterferingReads || !Acc.isRead()))
+ return true;
+
+ bool Dominates = FindInterferingWrites && DT && Exact &&
+ Acc.isMustAccess() && AccInSameScope &&
+ DT->dominates(Acc.getRemoteInst(), &I);
+ if (Dominates)
+ DominatingWrites.insert(&Acc);
+
+ // Track if all interesting accesses are in the same `nosync` function as
+ // the given instruction.
+ AllInSameNoSyncFn &= Acc.getRemoteInst()->getFunction() == &Scope;
+
+ InterferingAccesses.push_back({&Acc, Exact});
+ return true;
+ };
+ if (!State::forallInterferingAccesses(I, AccessCB, Range))
+ return false;
+
+ HasBeenWrittenTo = !DominatingWrites.empty();
+
+ // Dominating writes form a chain, find the least/lowest member.
+ Instruction *LeastDominatingWriteInst = nullptr;
+ for (const Access *Acc : DominatingWrites) {
+ if (!LeastDominatingWriteInst) {
+ LeastDominatingWriteInst = Acc->getRemoteInst();
+ } else if (DT->dominates(LeastDominatingWriteInst,
+ Acc->getRemoteInst())) {
+ LeastDominatingWriteInst = Acc->getRemoteInst();
+ }
+ }
+
+ // Helper to determine if we can skip a specific write access.
+ auto CanSkipAccess = [&](const Access &Acc, bool Exact) {
+ if (SkipCB && SkipCB(Acc))
+ return true;
+ if (!CanIgnoreThreading(Acc))
+ return false;
+
+ // Check read (RAW) dependences and write (WAR) dependences as necessary.
+ // If we successfully excluded all effects we are interested in, the
+ // access can be skipped.
+ bool ReadChecked = !FindInterferingReads;
+ bool WriteChecked = !FindInterferingWrites;
+
+ // If the instruction cannot reach the access, the former does not
+ // interfere with what the access reads.
+ if (!ReadChecked) {
+ if (!AA::isPotentiallyReachable(A, I, *Acc.getRemoteInst(), QueryingAA,
+ &ExclusionSet, IsLiveInCalleeCB))
+ ReadChecked = true;
+ }
+ // If the instruction cannot be reach from the access, the latter does not
+ // interfere with what the instruction reads.
+ if (!WriteChecked) {
+ if (!AA::isPotentiallyReachable(A, *Acc.getRemoteInst(), I, QueryingAA,
+ &ExclusionSet, IsLiveInCalleeCB))
+ WriteChecked = true;
+ }
+
+ // If we still might be affected by the write of the access but there are
+ // dominating writes in the function of the instruction
+ // (HasBeenWrittenTo), we can try to reason that the access is overwritten
+ // by them. This would have happend above if they are all in the same
+ // function, so we only check the inter-procedural case. Effectively, we
+ // want to show that there is no call after the dominting write that might
+ // reach the access, and when it returns reach the instruction with the
+ // updated value. To this end, we iterate all call sites, check if they
+ // might reach the instruction without going through another access
+ // (ExclusionSet) and at the same time might reach the access. However,
+ // that is all part of AAInterFnReachability.
+ if (!WriteChecked && HasBeenWrittenTo &&
+ Acc.getRemoteInst()->getFunction() != &Scope) {
+
+ const auto *FnReachabilityAA = A.getAAFor<AAInterFnReachability>(
+ QueryingAA, IRPosition::function(Scope), DepClassTy::OPTIONAL);
+
+ // Without going backwards in the call tree, can we reach the access
+ // from the least dominating write. Do not allow to pass the instruction
+ // itself either.
+ bool Inserted = ExclusionSet.insert(&I).second;
+
+ if (!FnReachabilityAA ||
+ !FnReachabilityAA->instructionCanReach(
+ A, *LeastDominatingWriteInst,
+ *Acc.getRemoteInst()->getFunction(), &ExclusionSet))
+ WriteChecked = true;
+
+ if (Inserted)
+ ExclusionSet.erase(&I);
+ }
+
+ if (ReadChecked && WriteChecked)
+ return true;
+
+ if (!DT || !UseDominanceReasoning)
+ return false;
+ if (!DominatingWrites.count(&Acc))
+ return false;
+ return LeastDominatingWriteInst != Acc.getRemoteInst();
+ };
+
+ // Run the user callback on all accesses we cannot skip and return if
+ // that succeeded for all or not.
+ for (auto &It : InterferingAccesses) {
+ if ((!AllInSameNoSyncFn && !IsThreadLocalObj && !ExecDomainAA) ||
+ !CanSkipAccess(*It.first, It.second)) {
+ if (!UserCB(*It.first, It.second))
+ return false;
+ }
+ }
+ return true;
+ }
+
+ ChangeStatus translateAndAddStateFromCallee(Attributor &A,
+ const AAPointerInfo &OtherAA,
+ CallBase &CB) {
+ using namespace AA::PointerInfo;
+ if (!OtherAA.getState().isValidState() || !isValidState())
+ return indicatePessimisticFixpoint();
+
+ const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
+ bool IsByval = OtherAAImpl.getAssociatedArgument()->hasByValAttr();
+
+ // Combine the accesses bin by bin.
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ const auto &State = OtherAAImpl.getState();
+ for (const auto &It : State) {
+ for (auto Index : It.getSecond()) {
+ const auto &RAcc = State.getAccess(Index);
+ if (IsByval && !RAcc.isRead())
+ continue;
+ bool UsedAssumedInformation = false;
+ AccessKind AK = RAcc.getKind();
+ auto Content = A.translateArgumentToCallSiteContent(
+ RAcc.getContent(), CB, *this, UsedAssumedInformation);
+ AK = AccessKind(AK & (IsByval ? AccessKind::AK_R : AccessKind::AK_RW));
+ AK = AccessKind(AK | (RAcc.isMayAccess() ? AK_MAY : AK_MUST));
+
+ Changed |= addAccess(A, RAcc.getRanges(), CB, Content, AK,
+ RAcc.getType(), RAcc.getRemoteInst());
+ }
+ }
+ return Changed;
+ }
+
+ ChangeStatus translateAndAddState(Attributor &A, const AAPointerInfo &OtherAA,
+ const OffsetInfo &Offsets, CallBase &CB) {
+ using namespace AA::PointerInfo;
+ if (!OtherAA.getState().isValidState() || !isValidState())
+ return indicatePessimisticFixpoint();
+
+ const auto &OtherAAImpl = static_cast<const AAPointerInfoImpl &>(OtherAA);
+
+ // Combine the accesses bin by bin.
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ const auto &State = OtherAAImpl.getState();
+ for (const auto &It : State) {
+ for (auto Index : It.getSecond()) {
+ const auto &RAcc = State.getAccess(Index);
+ for (auto Offset : Offsets) {
+ auto NewRanges = Offset == AA::RangeTy::Unknown
+ ? AA::RangeTy::getUnknown()
+ : RAcc.getRanges();
+ if (!NewRanges.isUnknown()) {
+ NewRanges.addToAllOffsets(Offset);
+ }
+ Changed |=
+ addAccess(A, NewRanges, CB, RAcc.getContent(), RAcc.getKind(),
+ RAcc.getType(), RAcc.getRemoteInst());
+ }
+ }
+ }
+ return Changed;
+ }
+
+ /// Statistic tracking for all AAPointerInfo implementations.
+ /// See AbstractAttribute::trackStatistics().
+ void trackPointerInfoStatistics(const IRPosition &IRP) const {}
+
+ /// Dump the state into \p O.
+ virtual void dumpState(raw_ostream &O) const override {
+ for (auto &It : OffsetBins) {
+ O << "[" << It.first.Offset << "-" << It.first.Offset + It.first.Size
+ << "] : " << It.getSecond().size() << "\n";
+ for (auto AccIndex : It.getSecond()) {
+ auto &Acc = AccessList[AccIndex];
+ O << " - " << Acc.getKind() << " - " << *Acc.getLocalInst() << "\n";
+ if (Acc.getLocalInst() != Acc.getRemoteInst())
+ O << " --> " << *Acc.getRemoteInst()
+ << "\n";
+ if (!Acc.isWrittenValueYetUndetermined()) {
+ if (isa_and_nonnull<Function>(Acc.getWrittenValue()))
+ O << " - c: func " << Acc.getWrittenValue()->getName()
+ << "\n";
+ else if (Acc.getWrittenValue())
+ O << " - c: " << *Acc.getWrittenValue() << "\n";
+ else
+ O << " - c: <unknown>\n";
+ }
+ }
+ }
+ }
+};
+
+struct AAPointerInfoFloating : public AAPointerInfoImpl {
+ using AccessKind = AAPointerInfo::AccessKind;
+ AAPointerInfoFloating(const IRPosition &IRP, Attributor &A)
+ : AAPointerInfoImpl(IRP, A) {}
+
+ /// Deal with an access and signal if it was handled successfully.
+ bool handleAccess(Attributor &A, Instruction &I,
+ std::optional<Value *> Content, AccessKind Kind,
+ SmallVectorImpl<int64_t> &Offsets, ChangeStatus &Changed,
+ Type &Ty) {
+ using namespace AA::PointerInfo;
+ auto Size = AA::RangeTy::Unknown;
+ const DataLayout &DL = A.getDataLayout();
+ TypeSize AccessSize = DL.getTypeStoreSize(&Ty);
+ if (!AccessSize.isScalable())
+ Size = AccessSize.getFixedValue();
+
+ // Make a strictly ascending list of offsets as required by addAccess()
+ llvm::sort(Offsets);
+ auto *Last = std::unique(Offsets.begin(), Offsets.end());
+ Offsets.erase(Last, Offsets.end());
+
+ VectorType *VT = dyn_cast<VectorType>(&Ty);
+ if (!VT || VT->getElementCount().isScalable() ||
+ !Content.value_or(nullptr) || !isa<Constant>(*Content) ||
+ (*Content)->getType() != VT ||
+ DL.getTypeStoreSize(VT->getElementType()).isScalable()) {
+ Changed = Changed | addAccess(A, {Offsets, Size}, I, Content, Kind, &Ty);
+ } else {
+ // Handle vector stores with constant content element-wise.
+ // TODO: We could look for the elements or create instructions
+ // representing them.
+ // TODO: We need to push the Content into the range abstraction
+ // (AA::RangeTy) to allow different content values for different
+ // ranges. ranges. Hence, support vectors storing different values.
+ Type *ElementType = VT->getElementType();
+ int64_t ElementSize = DL.getTypeStoreSize(ElementType).getFixedValue();
+ auto *ConstContent = cast<Constant>(*Content);
+ Type *Int32Ty = Type::getInt32Ty(ElementType->getContext());
+ SmallVector<int64_t> ElementOffsets(Offsets.begin(), Offsets.end());
+
+ for (int i = 0, e = VT->getElementCount().getFixedValue(); i != e; ++i) {
+ Value *ElementContent = ConstantExpr::getExtractElement(
+ ConstContent, ConstantInt::get(Int32Ty, i));
+
+ // Add the element access.
+ Changed = Changed | addAccess(A, {ElementOffsets, ElementSize}, I,
+ ElementContent, Kind, ElementType);
+
+ // Advance the offsets for the next element.
+ for (auto &ElementOffset : ElementOffsets)
+ ElementOffset += ElementSize;
+ }
+ }
+ return true;
+ };
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override;
+
+ /// If the indices to \p GEP can be traced to constants, incorporate all
+ /// of these into \p UsrOI.
+ ///
+ /// \return true iff \p UsrOI is updated.
+ bool collectConstantsForGEP(Attributor &A, const DataLayout &DL,
+ OffsetInfo &UsrOI, const OffsetInfo &PtrOI,
+ const GEPOperator *GEP);
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
+ }
+};
+
+bool AAPointerInfoFloating::collectConstantsForGEP(Attributor &A,
+ const DataLayout &DL,
+ OffsetInfo &UsrOI,
+ const OffsetInfo &PtrOI,
+ const GEPOperator *GEP) {
+ unsigned BitWidth = DL.getIndexTypeSizeInBits(GEP->getType());
+ MapVector<Value *, APInt> VariableOffsets;
+ APInt ConstantOffset(BitWidth, 0);
+
+ assert(!UsrOI.isUnknown() && !PtrOI.isUnknown() &&
+ "Don't look for constant values if the offset has already been "
+ "determined to be unknown.");
+
+ if (!GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
+ UsrOI.setUnknown();
+ return true;
+ }
+
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] GEP offset is "
+ << (VariableOffsets.empty() ? "" : "not") << " constant "
+ << *GEP << "\n");
+
+ auto Union = PtrOI;
+ Union.addToAll(ConstantOffset.getSExtValue());
+
+ // Each VI in VariableOffsets has a set of potential constant values. Every
+ // combination of elements, picked one each from these sets, is separately
+ // added to the original set of offsets, thus resulting in more offsets.
+ for (const auto &VI : VariableOffsets) {
+ auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
+ *this, IRPosition::value(*VI.first), DepClassTy::OPTIONAL);
+ if (!PotentialConstantsAA || !PotentialConstantsAA->isValidState()) {
+ UsrOI.setUnknown();
+ return true;
+ }
+
+ // UndefValue is treated as a zero, which leaves Union as is.
+ if (PotentialConstantsAA->undefIsContained())
+ continue;
+
+ // We need at least one constant in every set to compute an actual offset.
+ // Otherwise, we end up pessimizing AAPointerInfo by respecting offsets that
+ // don't actually exist. In other words, the absence of constant values
+ // implies that the operation can be assumed dead for now.
+ auto &AssumedSet = PotentialConstantsAA->getAssumedSet();
+ if (AssumedSet.empty())
+ return false;
+
+ OffsetInfo Product;
+ for (const auto &ConstOffset : AssumedSet) {
+ auto CopyPerOffset = Union;
+ CopyPerOffset.addToAll(ConstOffset.getSExtValue() *
+ VI.second.getZExtValue());
+ Product.merge(CopyPerOffset);
+ }
+ Union = Product;
+ }
+
+ UsrOI = std::move(Union);
+ return true;
+}
+
+ChangeStatus AAPointerInfoFloating::updateImpl(Attributor &A) {
+ using namespace AA::PointerInfo;
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ const DataLayout &DL = A.getDataLayout();
+ Value &AssociatedValue = getAssociatedValue();
+
+ DenseMap<Value *, OffsetInfo> OffsetInfoMap;
+ OffsetInfoMap[&AssociatedValue].insert(0);
+
+ auto HandlePassthroughUser = [&](Value *Usr, Value *CurPtr, bool &Follow) {
+ // One does not simply walk into a map and assign a reference to a possibly
+ // new location. That can cause an invalidation before the assignment
+ // happens, like so:
+ //
+ // OffsetInfoMap[Usr] = OffsetInfoMap[CurPtr]; /* bad idea! */
+ //
+ // The RHS is a reference that may be invalidated by an insertion caused by
+ // the LHS. So we ensure that the side-effect of the LHS happens first.
+ auto &UsrOI = OffsetInfoMap[Usr];
+ auto &PtrOI = OffsetInfoMap[CurPtr];
+ assert(!PtrOI.isUnassigned() &&
+ "Cannot pass through if the input Ptr was not visited!");
+ UsrOI = PtrOI;
+ Follow = true;
+ return true;
+ };
+
+ const auto *F = getAnchorScope();
+ const auto *CI =
+ F ? A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(*F)
+ : nullptr;
+ const auto *TLI =
+ F ? A.getInfoCache().getTargetLibraryInfoForFunction(*F) : nullptr;
+
+ auto UsePred = [&](const Use &U, bool &Follow) -> bool {
+ Value *CurPtr = U.get();
+ User *Usr = U.getUser();
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] Analyze " << *CurPtr << " in " << *Usr
+ << "\n");
+ assert(OffsetInfoMap.count(CurPtr) &&
+ "The current pointer offset should have been seeded!");
+
+ if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Usr)) {
+ if (CE->isCast())
+ return HandlePassthroughUser(Usr, CurPtr, Follow);
+ if (CE->isCompare())
+ return true;
+ if (!isa<GEPOperator>(CE)) {
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled constant user " << *CE
+ << "\n");
+ return false;
+ }
+ }
+ if (auto *GEP = dyn_cast<GEPOperator>(Usr)) {
+ // Note the order here, the Usr access might change the map, CurPtr is
+ // already in it though.
+ auto &UsrOI = OffsetInfoMap[Usr];
+ auto &PtrOI = OffsetInfoMap[CurPtr];
+
+ if (UsrOI.isUnknown())
+ return true;
+
+ if (PtrOI.isUnknown()) {
+ Follow = true;
+ UsrOI.setUnknown();
+ return true;
+ }
+
+ Follow = collectConstantsForGEP(A, DL, UsrOI, PtrOI, GEP);
+ return true;
+ }
+ if (isa<PtrToIntInst>(Usr))
+ return false;
+ if (isa<CastInst>(Usr) || isa<SelectInst>(Usr) || isa<ReturnInst>(Usr))
+ return HandlePassthroughUser(Usr, CurPtr, Follow);
+
+ // For PHIs we need to take care of the recurrence explicitly as the value
+ // might change while we iterate through a loop. For now, we give up if
+ // the PHI is not invariant.
+ if (isa<PHINode>(Usr)) {
+ // Note the order here, the Usr access might change the map, CurPtr is
+ // already in it though.
+ bool IsFirstPHIUser = !OffsetInfoMap.count(Usr);
+ auto &UsrOI = OffsetInfoMap[Usr];
+ auto &PtrOI = OffsetInfoMap[CurPtr];
+
+ // Check if the PHI operand has already an unknown offset as we can't
+ // improve on that anymore.
+ if (PtrOI.isUnknown()) {
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand offset unknown "
+ << *CurPtr << " in " << *Usr << "\n");
+ Follow = !UsrOI.isUnknown();
+ UsrOI.setUnknown();
+ return true;
+ }
+
+ // Check if the PHI is invariant (so far).
+ if (UsrOI == PtrOI) {
+ assert(!PtrOI.isUnassigned() &&
+ "Cannot assign if the current Ptr was not visited!");
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant (so far)");
+ return true;
+ }
+
+ // Check if the PHI operand can be traced back to AssociatedValue.
+ APInt Offset(
+ DL.getIndexSizeInBits(CurPtr->getType()->getPointerAddressSpace()),
+ 0);
+ Value *CurPtrBase = CurPtr->stripAndAccumulateConstantOffsets(
+ DL, Offset, /* AllowNonInbounds */ true);
+ auto It = OffsetInfoMap.find(CurPtrBase);
+ if (It == OffsetInfoMap.end()) {
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI operand is too complex "
+ << *CurPtr << " in " << *Usr << "\n");
+ UsrOI.setUnknown();
+ Follow = true;
+ return true;
+ }
+
+ // Check if the PHI operand is not dependent on the PHI itself. Every
+ // recurrence is a cyclic net of PHIs in the data flow, and has an
+ // equivalent Cycle in the control flow. One of those PHIs must be in the
+ // header of that control flow Cycle. This is independent of the choice of
+ // Cycles reported by CycleInfo. It is sufficient to check the PHIs in
+ // every Cycle header; if such a node is marked unknown, this will
+ // eventually propagate through the whole net of PHIs in the recurrence.
+ if (mayBeInCycle(CI, cast<Instruction>(Usr), /* HeaderOnly */ true)) {
+ auto BaseOI = It->getSecond();
+ BaseOI.addToAll(Offset.getZExtValue());
+ if (IsFirstPHIUser || BaseOI == UsrOI) {
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] PHI is invariant " << *CurPtr
+ << " in " << *Usr << "\n");
+ return HandlePassthroughUser(Usr, CurPtr, Follow);
+ }
+
+ LLVM_DEBUG(
+ dbgs() << "[AAPointerInfo] PHI operand pointer offset mismatch "
+ << *CurPtr << " in " << *Usr << "\n");
+ UsrOI.setUnknown();
+ Follow = true;
+ return true;
+ }
+
+ UsrOI.merge(PtrOI);
+ Follow = true;
+ return true;
+ }
+
+ if (auto *LoadI = dyn_cast<LoadInst>(Usr)) {
+ // If the access is to a pointer that may or may not be the associated
+ // value, e.g. due to a PHI, we cannot assume it will be read.
+ AccessKind AK = AccessKind::AK_R;
+ if (getUnderlyingObject(CurPtr) == &AssociatedValue)
+ AK = AccessKind(AK | AccessKind::AK_MUST);
+ else
+ AK = AccessKind(AK | AccessKind::AK_MAY);
+ if (!handleAccess(A, *LoadI, /* Content */ nullptr, AK,
+ OffsetInfoMap[CurPtr].Offsets, Changed,
+ *LoadI->getType()))
+ return false;
+
+ auto IsAssumption = [](Instruction &I) {
+ if (auto *II = dyn_cast<IntrinsicInst>(&I))
+ return II->isAssumeLikeIntrinsic();
+ return false;
+ };
+
+ auto IsImpactedInRange = [&](Instruction *FromI, Instruction *ToI) {
+ // Check if the assumption and the load are executed together without
+ // memory modification.
+ do {
+ if (FromI->mayWriteToMemory() && !IsAssumption(*FromI))
+ return true;
+ FromI = FromI->getNextNonDebugInstruction();
+ } while (FromI && FromI != ToI);
+ return false;
+ };
+
+ BasicBlock *BB = LoadI->getParent();
+ auto IsValidAssume = [&](IntrinsicInst &IntrI) {
+ if (IntrI.getIntrinsicID() != Intrinsic::assume)
+ return false;
+ BasicBlock *IntrBB = IntrI.getParent();
+ if (IntrI.getParent() == BB) {
+ if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(), &IntrI))
+ return false;
+ } else {
+ auto PredIt = pred_begin(IntrBB);
+ if (PredIt == pred_end(IntrBB))
+ return false;
+ if ((*PredIt) != BB)
+ return false;
+ if (++PredIt != pred_end(IntrBB))
+ return false;
+ for (auto *SuccBB : successors(BB)) {
+ if (SuccBB == IntrBB)
+ continue;
+ if (isa<UnreachableInst>(SuccBB->getTerminator()))
+ continue;
+ return false;
+ }
+ if (IsImpactedInRange(LoadI->getNextNonDebugInstruction(),
+ BB->getTerminator()))
+ return false;
+ if (IsImpactedInRange(&IntrBB->front(), &IntrI))
+ return false;
+ }
+ return true;
+ };
+
+ std::pair<Value *, IntrinsicInst *> Assumption;
+ for (const Use &LoadU : LoadI->uses()) {
+ if (auto *CmpI = dyn_cast<CmpInst>(LoadU.getUser())) {
+ if (!CmpI->isEquality() || !CmpI->isTrueWhenEqual())
+ continue;
+ for (const Use &CmpU : CmpI->uses()) {
+ if (auto *IntrI = dyn_cast<IntrinsicInst>(CmpU.getUser())) {
+ if (!IsValidAssume(*IntrI))
+ continue;
+ int Idx = CmpI->getOperandUse(0) == LoadU;
+ Assumption = {CmpI->getOperand(Idx), IntrI};
+ break;
+ }
+ }
+ }
+ if (Assumption.first)
+ break;
+ }
+
+ // Check if we found an assumption associated with this load.
+ if (!Assumption.first || !Assumption.second)
+ return true;
+
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] Assumption found "
+ << *Assumption.second << ": " << *LoadI
+ << " == " << *Assumption.first << "\n");
+ bool UsedAssumedInformation = false;
+ std::optional<Value *> Content = nullptr;
+ if (Assumption.first)
+ Content =
+ A.getAssumedSimplified(*Assumption.first, *this,
+ UsedAssumedInformation, AA::Interprocedural);
+ return handleAccess(
+ A, *Assumption.second, Content, AccessKind::AK_ASSUMPTION,
+ OffsetInfoMap[CurPtr].Offsets, Changed, *LoadI->getType());
+ }
+
+ auto HandleStoreLike = [&](Instruction &I, Value *ValueOp, Type &ValueTy,
+ ArrayRef<Value *> OtherOps, AccessKind AK) {
+ for (auto *OtherOp : OtherOps) {
+ if (OtherOp == CurPtr) {
+ LLVM_DEBUG(
+ dbgs()
+ << "[AAPointerInfo] Escaping use in store like instruction " << I
+ << "\n");
+ return false;
+ }
+ }
+
+ // If the access is to a pointer that may or may not be the associated
+ // value, e.g. due to a PHI, we cannot assume it will be written.
+ if (getUnderlyingObject(CurPtr) == &AssociatedValue)
+ AK = AccessKind(AK | AccessKind::AK_MUST);
+ else
+ AK = AccessKind(AK | AccessKind::AK_MAY);
+ bool UsedAssumedInformation = false;
+ std::optional<Value *> Content = nullptr;
+ if (ValueOp)
+ Content = A.getAssumedSimplified(
+ *ValueOp, *this, UsedAssumedInformation, AA::Interprocedural);
+ return handleAccess(A, I, Content, AK, OffsetInfoMap[CurPtr].Offsets,
+ Changed, ValueTy);
+ };
+
+ if (auto *StoreI = dyn_cast<StoreInst>(Usr))
+ return HandleStoreLike(*StoreI, StoreI->getValueOperand(),
+ *StoreI->getValueOperand()->getType(),
+ {StoreI->getValueOperand()}, AccessKind::AK_W);
+ if (auto *RMWI = dyn_cast<AtomicRMWInst>(Usr))
+ return HandleStoreLike(*RMWI, nullptr, *RMWI->getValOperand()->getType(),
+ {RMWI->getValOperand()}, AccessKind::AK_RW);
+ if (auto *CXI = dyn_cast<AtomicCmpXchgInst>(Usr))
+ return HandleStoreLike(
+ *CXI, nullptr, *CXI->getNewValOperand()->getType(),
+ {CXI->getCompareOperand(), CXI->getNewValOperand()},
+ AccessKind::AK_RW);
+
+ if (auto *CB = dyn_cast<CallBase>(Usr)) {
+ if (CB->isLifetimeStartOrEnd())
+ return true;
+ if (getFreedOperand(CB, TLI) == U)
+ return true;
+ if (CB->isArgOperand(&U)) {
+ unsigned ArgNo = CB->getArgOperandNo(&U);
+ const auto *CSArgPI = A.getAAFor<AAPointerInfo>(
+ *this, IRPosition::callsite_argument(*CB, ArgNo),
+ DepClassTy::REQUIRED);
+ if (!CSArgPI)
+ return false;
+ Changed =
+ translateAndAddState(A, *CSArgPI, OffsetInfoMap[CurPtr], *CB) |
+ Changed;
+ return isValidState();
+ }
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] Call user not handled " << *CB
+ << "\n");
+ // TODO: Allow some call uses
+ return false;
+ }
+
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] User not handled " << *Usr << "\n");
+ return false;
+ };
+ auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
+ assert(OffsetInfoMap.count(OldU) && "Old use should be known already!");
+ if (OffsetInfoMap.count(NewU)) {
+ LLVM_DEBUG({
+ if (!(OffsetInfoMap[NewU] == OffsetInfoMap[OldU])) {
+ dbgs() << "[AAPointerInfo] Equivalent use callback failed: "
+ << OffsetInfoMap[NewU] << " vs " << OffsetInfoMap[OldU]
+ << "\n";
+ }
+ });
+ return OffsetInfoMap[NewU] == OffsetInfoMap[OldU];
+ }
+ OffsetInfoMap[NewU] = OffsetInfoMap[OldU];
+ return true;
+ };
+ if (!A.checkForAllUses(UsePred, *this, AssociatedValue,
+ /* CheckBBLivenessOnly */ true, DepClassTy::OPTIONAL,
+ /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] Check for all uses failed, abort!\n");
+ return indicatePessimisticFixpoint();
+ }
+
+ LLVM_DEBUG({
+ dbgs() << "Accesses by bin after update:\n";
+ dumpState(dbgs());
+ });
+
+ return Changed;
+}
+
+struct AAPointerInfoReturned final : AAPointerInfoImpl {
+ AAPointerInfoReturned(const IRPosition &IRP, Attributor &A)
+ : AAPointerInfoImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ return indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
+ }
+};
+
+struct AAPointerInfoArgument final : AAPointerInfoFloating {
+ AAPointerInfoArgument(const IRPosition &IRP, Attributor &A)
+ : AAPointerInfoFloating(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
+ }
+};
+
+struct AAPointerInfoCallSiteArgument final : AAPointerInfoFloating {
+ AAPointerInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAPointerInfoFloating(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ using namespace AA::PointerInfo;
+ // We handle memory intrinsics explicitly, at least the first (=
+ // destination) and second (=source) arguments as we know how they are
+ // accessed.
+ if (auto *MI = dyn_cast_or_null<MemIntrinsic>(getCtxI())) {
+ ConstantInt *Length = dyn_cast<ConstantInt>(MI->getLength());
+ int64_t LengthVal = AA::RangeTy::Unknown;
+ if (Length)
+ LengthVal = Length->getSExtValue();
+ unsigned ArgNo = getIRPosition().getCallSiteArgNo();
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ if (ArgNo > 1) {
+ LLVM_DEBUG(dbgs() << "[AAPointerInfo] Unhandled memory intrinsic "
+ << *MI << "\n");
+ return indicatePessimisticFixpoint();
+ } else {
+ auto Kind =
+ ArgNo == 0 ? AccessKind::AK_MUST_WRITE : AccessKind::AK_MUST_READ;
+ Changed =
+ Changed | addAccess(A, {0, LengthVal}, *MI, nullptr, Kind, nullptr);
+ }
+ LLVM_DEBUG({
+ dbgs() << "Accesses by bin after update:\n";
+ dumpState(dbgs());
+ });
+
+ return Changed;
+ }
+
+ // TODO: Once we have call site specific value information we can provide
+ // call site specific liveness information and then it makes
+ // sense to specialize attributes for call sites arguments instead of
+ // redirecting requests to the callee argument.
+ Argument *Arg = getAssociatedArgument();
+ if (Arg) {
+ const IRPosition &ArgPos = IRPosition::argument(*Arg);
+ auto *ArgAA =
+ A.getAAFor<AAPointerInfo>(*this, ArgPos, DepClassTy::REQUIRED);
+ if (ArgAA && ArgAA->getState().isValidState())
+ return translateAndAddStateFromCallee(A, *ArgAA,
+ *cast<CallBase>(getCtxI()));
+ if (!Arg->getParent()->isDeclaration())
+ return indicatePessimisticFixpoint();
+ }
+
+ bool IsKnownNoCapture;
+ if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoCapture))
+ return indicatePessimisticFixpoint();
+
+ bool IsKnown = false;
+ if (AA::isAssumedReadNone(A, getIRPosition(), *this, IsKnown))
+ return ChangeStatus::UNCHANGED;
+ bool ReadOnly = AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown);
+ auto Kind =
+ ReadOnly ? AccessKind::AK_MAY_READ : AccessKind::AK_MAY_READ_WRITE;
+ return addAccess(A, AA::RangeTy::getUnknown(), *getCtxI(), nullptr, Kind,
+ nullptr);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
+ }
+};
+
+struct AAPointerInfoCallSiteReturned final : AAPointerInfoFloating {
+ AAPointerInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AAPointerInfoFloating(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ AAPointerInfoImpl::trackPointerInfoStatistics(getIRPosition());
+ }
+};
+} // namespace
+
+/// -----------------------NoUnwind Function Attribute--------------------------
+
+namespace {
+struct AANoUnwindImpl : AANoUnwind {
+ AANoUnwindImpl(const IRPosition &IRP, Attributor &A) : AANoUnwind(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ bool IsKnown;
+ assert(!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
+ A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
+ (void)IsKnown;
+ }
+
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "nounwind" : "may-unwind";
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto Opcodes = {
+ (unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
+ (unsigned)Instruction::Call, (unsigned)Instruction::CleanupRet,
+ (unsigned)Instruction::CatchSwitch, (unsigned)Instruction::Resume};
+
+ auto CheckForNoUnwind = [&](Instruction &I) {
+ if (!I.mayThrow(/* IncludePhaseOneUnwind */ true))
+ return true;
+
+ if (const auto *CB = dyn_cast<CallBase>(&I)) {
+ bool IsKnownNoUnwind;
+ return AA::hasAssumedIRAttr<Attribute::NoUnwind>(
+ A, this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED,
+ IsKnownNoUnwind);
+ }
+ return false;
+ };
+
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllInstructions(CheckForNoUnwind, *this, Opcodes,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ return ChangeStatus::UNCHANGED;
+ }
+};
+
+struct AANoUnwindFunction final : public AANoUnwindImpl {
+ AANoUnwindFunction(const IRPosition &IRP, Attributor &A)
+ : AANoUnwindImpl(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nounwind) }
+};
+
+/// NoUnwind attribute deduction for a call sites.
+struct AANoUnwindCallSite final
+ : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl> {
+ AANoUnwindCallSite(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AANoUnwind, AANoUnwindImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nounwind); }
+};
+} // namespace
+
+/// ------------------------ NoSync Function Attribute -------------------------
+
+bool AANoSync::isAlignedBarrier(const CallBase &CB, bool ExecutedAligned) {
+ switch (CB.getIntrinsicID()) {
+ case Intrinsic::nvvm_barrier0:
+ case Intrinsic::nvvm_barrier0_and:
+ case Intrinsic::nvvm_barrier0_or:
+ case Intrinsic::nvvm_barrier0_popc:
+ return true;
+ case Intrinsic::amdgcn_s_barrier:
+ if (ExecutedAligned)
+ return true;
+ break;
+ default:
+ break;
+ }
+ return hasAssumption(CB, KnownAssumptionString("ompx_aligned_barrier"));
+}
+
+bool AANoSync::isNonRelaxedAtomic(const Instruction *I) {
+ if (!I->isAtomic())
+ return false;
+
+ if (auto *FI = dyn_cast<FenceInst>(I))
+ // All legal orderings for fence are stronger than monotonic.
+ return FI->getSyncScopeID() != SyncScope::SingleThread;
+ if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
+ // Unordered is not a legal ordering for cmpxchg.
+ return (AI->getSuccessOrdering() != AtomicOrdering::Monotonic ||
+ AI->getFailureOrdering() != AtomicOrdering::Monotonic);
+ }
+
+ AtomicOrdering Ordering;
+ switch (I->getOpcode()) {
+ case Instruction::AtomicRMW:
+ Ordering = cast<AtomicRMWInst>(I)->getOrdering();
+ break;
+ case Instruction::Store:
+ Ordering = cast<StoreInst>(I)->getOrdering();
+ break;
+ case Instruction::Load:
+ Ordering = cast<LoadInst>(I)->getOrdering();
+ break;
+ default:
+ llvm_unreachable(
+ "New atomic operations need to be known in the attributor.");
+ }
+
+ return (Ordering != AtomicOrdering::Unordered &&
+ Ordering != AtomicOrdering::Monotonic);
+}
+
+/// Return true if this intrinsic is nosync. This is only used for intrinsics
+/// which would be nosync except that they have a volatile flag. All other
+/// intrinsics are simply annotated with the nosync attribute in Intrinsics.td.
+bool AANoSync::isNoSyncIntrinsic(const Instruction *I) {
+ if (auto *MI = dyn_cast<MemIntrinsic>(I))
+ return !MI->isVolatile();
+ return false;
+}
+
+namespace {
+struct AANoSyncImpl : AANoSync {
+ AANoSyncImpl(const IRPosition &IRP, Attributor &A) : AANoSync(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ bool IsKnown;
+ assert(!AA::hasAssumedIRAttr<Attribute::NoSync>(A, nullptr, getIRPosition(),
+ DepClassTy::NONE, IsKnown));
+ (void)IsKnown;
+ }
+
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "nosync" : "may-sync";
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override;
+};
+
+ChangeStatus AANoSyncImpl::updateImpl(Attributor &A) {
+
+ auto CheckRWInstForNoSync = [&](Instruction &I) {
+ return AA::isNoSyncInst(A, I, *this);
+ };
+
+ auto CheckForNoSync = [&](Instruction &I) {
+ // At this point we handled all read/write effects and they are all
+ // nosync, so they can be skipped.
+ if (I.mayReadOrWriteMemory())
+ return true;
+
+ bool IsKnown;
+ CallBase &CB = cast<CallBase>(I);
+ if (AA::hasAssumedIRAttr<Attribute::NoSync>(
+ A, this, IRPosition::callsite_function(CB), DepClassTy::OPTIONAL,
+ IsKnown))
+ return true;
+
+ // non-convergent and readnone imply nosync.
+ return !CB.isConvergent();
+ };
+
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllReadWriteInstructions(CheckRWInstForNoSync, *this,
+ UsedAssumedInformation) ||
+ !A.checkForAllCallLikeInstructions(CheckForNoSync, *this,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ return ChangeStatus::UNCHANGED;
+}
+
+struct AANoSyncFunction final : public AANoSyncImpl {
+ AANoSyncFunction(const IRPosition &IRP, Attributor &A)
+ : AANoSyncImpl(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nosync) }
+};
+
+/// NoSync attribute deduction for a call sites.
+struct AANoSyncCallSite final : AACalleeToCallSite<AANoSync, AANoSyncImpl> {
+ AANoSyncCallSite(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AANoSync, AANoSyncImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nosync); }
+};
+} // namespace
+
+/// ------------------------ No-Free Attributes ----------------------------
+
+namespace {
+struct AANoFreeImpl : public AANoFree {
+ AANoFreeImpl(const IRPosition &IRP, Attributor &A) : AANoFree(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ bool IsKnown;
+ assert(!AA::hasAssumedIRAttr<Attribute::NoFree>(A, nullptr, getIRPosition(),
+ DepClassTy::NONE, IsKnown));
+ (void)IsKnown;
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto CheckForNoFree = [&](Instruction &I) {
+ bool IsKnown;
+ return AA::hasAssumedIRAttr<Attribute::NoFree>(
+ A, this, IRPosition::callsite_function(cast<CallBase>(I)),
+ DepClassTy::REQUIRED, IsKnown);
+ };
+
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllCallLikeInstructions(CheckForNoFree, *this,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "nofree" : "may-free";
+ }
+};
+
+struct AANoFreeFunction final : public AANoFreeImpl {
+ AANoFreeFunction(const IRPosition &IRP, Attributor &A)
+ : AANoFreeImpl(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(nofree) }
+};
+
+/// NoFree attribute deduction for a call sites.
+struct AANoFreeCallSite final : AACalleeToCallSite<AANoFree, AANoFreeImpl> {
+ AANoFreeCallSite(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AANoFree, AANoFreeImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(nofree); }
+};
+
+/// NoFree attribute for floating values.
+struct AANoFreeFloating : AANoFreeImpl {
+ AANoFreeFloating(const IRPosition &IRP, Attributor &A)
+ : AANoFreeImpl(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override{STATS_DECLTRACK_FLOATING_ATTR(nofree)}
+
+ /// See Abstract Attribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ const IRPosition &IRP = getIRPosition();
+
+ bool IsKnown;
+ if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this,
+ IRPosition::function_scope(IRP),
+ DepClassTy::OPTIONAL, IsKnown))
+ return ChangeStatus::UNCHANGED;
+
+ Value &AssociatedValue = getIRPosition().getAssociatedValue();
+ auto Pred = [&](const Use &U, bool &Follow) -> bool {
+ Instruction *UserI = cast<Instruction>(U.getUser());
+ if (auto *CB = dyn_cast<CallBase>(UserI)) {
+ if (CB->isBundleOperand(&U))
+ return false;
+ if (!CB->isArgOperand(&U))
+ return true;
+ unsigned ArgNo = CB->getArgOperandNo(&U);
+
+ bool IsKnown;
+ return AA::hasAssumedIRAttr<Attribute::NoFree>(
+ A, this, IRPosition::callsite_argument(*CB, ArgNo),
+ DepClassTy::REQUIRED, IsKnown);
+ }
+
+ if (isa<GetElementPtrInst>(UserI) || isa<BitCastInst>(UserI) ||
+ isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
+ Follow = true;
+ return true;
+ }
+ if (isa<StoreInst>(UserI) || isa<LoadInst>(UserI) ||
+ isa<ReturnInst>(UserI))
+ return true;
+
+ // Unknown user.
+ return false;
+ };
+ if (!A.checkForAllUses(Pred, *this, AssociatedValue))
+ return indicatePessimisticFixpoint();
+
+ return ChangeStatus::UNCHANGED;
+ }
+};
+
+/// NoFree attribute for a call site argument.
+struct AANoFreeArgument final : AANoFreeFloating {
+ AANoFreeArgument(const IRPosition &IRP, Attributor &A)
+ : AANoFreeFloating(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofree) }
+};
+
+/// NoFree attribute for call site arguments.
+struct AANoFreeCallSiteArgument final : AANoFreeFloating {
+ AANoFreeCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AANoFreeFloating(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // TODO: Once we have call site specific value information we can provide
+ // call site specific liveness information and then it makes
+ // sense to specialize attributes for call sites arguments instead of
+ // redirecting requests to the callee argument.
+ Argument *Arg = getAssociatedArgument();
+ if (!Arg)
+ return indicatePessimisticFixpoint();
+ const IRPosition &ArgPos = IRPosition::argument(*Arg);
+ bool IsKnown;
+ if (AA::hasAssumedIRAttr<Attribute::NoFree>(A, this, ArgPos,
+ DepClassTy::REQUIRED, IsKnown))
+ return ChangeStatus::UNCHANGED;
+ return indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nofree)};
+};
+
+/// NoFree attribute for function return value.
+struct AANoFreeReturned final : AANoFreeFloating {
+ AANoFreeReturned(const IRPosition &IRP, Attributor &A)
+ : AANoFreeFloating(IRP, A) {
+ llvm_unreachable("NoFree is not applicable to function returns!");
+ }
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ llvm_unreachable("NoFree is not applicable to function returns!");
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ llvm_unreachable("NoFree is not applicable to function returns!");
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {}
+};
+
+/// NoFree attribute deduction for a call site return value.
+struct AANoFreeCallSiteReturned final : AANoFreeFloating {
+ AANoFreeCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AANoFreeFloating(IRP, A) {}
+
+ ChangeStatus manifest(Attributor &A) override {
+ return ChangeStatus::UNCHANGED;
+ }
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nofree) }
+};
+} // namespace
+
+/// ------------------------ NonNull Argument Attribute ------------------------
+
+bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions) {
+ SmallVector<Attribute::AttrKind, 2> AttrKinds;
+ AttrKinds.push_back(Attribute::NonNull);
+ if (!NullPointerIsDefined(IRP.getAnchorScope(),
+ IRP.getAssociatedType()->getPointerAddressSpace()))
+ AttrKinds.push_back(Attribute::Dereferenceable);
+ if (A.hasAttr(IRP, AttrKinds, IgnoreSubsumingPositions, Attribute::NonNull))
+ return true;
+
+ DominatorTree *DT = nullptr;
+ AssumptionCache *AC = nullptr;
+ InformationCache &InfoCache = A.getInfoCache();
+ if (const Function *Fn = IRP.getAnchorScope()) {
+ if (!Fn->isDeclaration()) {
+ DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*Fn);
+ AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*Fn);
+ }
+ }
+
+ SmallVector<AA::ValueAndContext> Worklist;
+ if (IRP.getPositionKind() != IRP_RETURNED) {
+ Worklist.push_back({IRP.getAssociatedValue(), IRP.getCtxI()});
+ } else {
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllInstructions(
+ [&](Instruction &I) {
+ Worklist.push_back({*cast<ReturnInst>(I).getReturnValue(), &I});
+ return true;
+ },
+ IRP.getAssociatedFunction(), nullptr, {Instruction::Ret},
+ UsedAssumedInformation))
+ return false;
+ }
+
+ if (llvm::any_of(Worklist, [&](AA::ValueAndContext VAC) {
+ return !isKnownNonZero(
+ VAC.getValue(),
+ SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
+ }))
+ return false;
+
+ A.manifestAttrs(IRP, {Attribute::get(IRP.getAnchorValue().getContext(),
+ Attribute::NonNull)});
+ return true;
+}
+
+namespace {
+static int64_t getKnownNonNullAndDerefBytesForUse(
+ Attributor &A, const AbstractAttribute &QueryingAA, Value &AssociatedValue,
+ const Use *U, const Instruction *I, bool &IsNonNull, bool &TrackUse) {
+ TrackUse = false;
+
+ const Value *UseV = U->get();
+ if (!UseV->getType()->isPointerTy())
+ return 0;
+
+ // We need to follow common pointer manipulation uses to the accesses they
+ // feed into. We can try to be smart to avoid looking through things we do not
+ // like for now, e.g., non-inbounds GEPs.
+ if (isa<CastInst>(I)) {
+ TrackUse = true;
+ return 0;
+ }
+
+ if (isa<GetElementPtrInst>(I)) {
+ TrackUse = true;
+ return 0;
+ }
+
+ Type *PtrTy = UseV->getType();
+ const Function *F = I->getFunction();
+ bool NullPointerIsDefined =
+ F ? llvm::NullPointerIsDefined(F, PtrTy->getPointerAddressSpace()) : true;
+ const DataLayout &DL = A.getInfoCache().getDL();
+ if (const auto *CB = dyn_cast<CallBase>(I)) {
+ if (CB->isBundleOperand(U)) {
+ if (RetainedKnowledge RK = getKnowledgeFromUse(
+ U, {Attribute::NonNull, Attribute::Dereferenceable})) {
+ IsNonNull |=
+ (RK.AttrKind == Attribute::NonNull || !NullPointerIsDefined);
+ return RK.ArgValue;
+ }
+ return 0;
+ }
+
+ if (CB->isCallee(U)) {
+ IsNonNull |= !NullPointerIsDefined;
+ return 0;
+ }
+
+ unsigned ArgNo = CB->getArgOperandNo(U);
+ IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
+ // As long as we only use known information there is no need to track
+ // dependences here.
+ bool IsKnownNonNull;
+ AA::hasAssumedIRAttr<Attribute::NonNull>(A, &QueryingAA, IRP,
+ DepClassTy::NONE, IsKnownNonNull);
+ IsNonNull |= IsKnownNonNull;
+ auto *DerefAA =
+ A.getAAFor<AADereferenceable>(QueryingAA, IRP, DepClassTy::NONE);
+ return DerefAA ? DerefAA->getKnownDereferenceableBytes() : 0;
+ }
+
+ std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);
+ if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() ||
+ Loc->Size.isScalable() || I->isVolatile())
+ return 0;
+
+ int64_t Offset;
+ const Value *Base =
+ getMinimalBaseOfPointer(A, QueryingAA, Loc->Ptr, Offset, DL);
+ if (Base && Base == &AssociatedValue) {
+ int64_t DerefBytes = Loc->Size.getValue() + Offset;
+ IsNonNull |= !NullPointerIsDefined;
+ return std::max(int64_t(0), DerefBytes);
+ }
+
+ /// Corner case when an offset is 0.
+ Base = GetPointerBaseWithConstantOffset(Loc->Ptr, Offset, DL,
+ /*AllowNonInbounds*/ true);
+ if (Base && Base == &AssociatedValue && Offset == 0) {
+ int64_t DerefBytes = Loc->Size.getValue();
+ IsNonNull |= !NullPointerIsDefined;
+ return std::max(int64_t(0), DerefBytes);
+ }
+
+ return 0;
+}
+
+struct AANonNullImpl : AANonNull {
+ AANonNullImpl(const IRPosition &IRP, Attributor &A) : AANonNull(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ Value &V = *getAssociatedValue().stripPointerCasts();
+ if (isa<ConstantPointerNull>(V)) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+
+ if (Instruction *CtxI = getCtxI())
+ followUsesInMBEC(*this, A, getState(), *CtxI);
+ }
+
+ /// See followUsesInMBEC
+ bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
+ AANonNull::StateType &State) {
+ bool IsNonNull = false;
+ bool TrackUse = false;
+ getKnownNonNullAndDerefBytesForUse(A, *this, getAssociatedValue(), U, I,
+ IsNonNull, TrackUse);
+ State.setKnown(IsNonNull);
+ return TrackUse;
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "nonnull" : "may-null";
+ }
+};
+
+/// NonNull attribute for a floating value.
+struct AANonNullFloating : public AANonNullImpl {
+ AANonNullFloating(const IRPosition &IRP, Attributor &A)
+ : AANonNullImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto CheckIRP = [&](const IRPosition &IRP) {
+ bool IsKnownNonNull;
+ return AA::hasAssumedIRAttr<Attribute::NonNull>(
+ A, *this, IRP, DepClassTy::OPTIONAL, IsKnownNonNull);
+ };
+
+ bool Stripped;
+ bool UsedAssumedInformation = false;
+ Value *AssociatedValue = &getAssociatedValue();
+ SmallVector<AA::ValueAndContext> Values;
+ if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
+ AA::AnyScope, UsedAssumedInformation))
+ Stripped = false;
+ else
+ Stripped =
+ Values.size() != 1 || Values.front().getValue() != AssociatedValue;
+
+ if (!Stripped) {
+ bool IsKnown;
+ if (auto *PHI = dyn_cast<PHINode>(AssociatedValue))
+ if (llvm::all_of(PHI->incoming_values(), [&](Value *Op) {
+ return AA::hasAssumedIRAttr<Attribute::NonNull>(
+ A, this, IRPosition::value(*Op), DepClassTy::OPTIONAL,
+ IsKnown);
+ }))
+ return ChangeStatus::UNCHANGED;
+ if (auto *Select = dyn_cast<SelectInst>(AssociatedValue))
+ if (AA::hasAssumedIRAttr<Attribute::NonNull>(
+ A, this, IRPosition::value(*Select->getFalseValue()),
+ DepClassTy::OPTIONAL, IsKnown) &&
+ AA::hasAssumedIRAttr<Attribute::NonNull>(
+ A, this, IRPosition::value(*Select->getTrueValue()),
+ DepClassTy::OPTIONAL, IsKnown))
+ return ChangeStatus::UNCHANGED;
+
+ // If we haven't stripped anything we might still be able to use a
+ // different AA, but only if the IRP changes. Effectively when we
+ // interpret this not as a call site value but as a floating/argument
+ // value.
+ const IRPosition AVIRP = IRPosition::value(*AssociatedValue);
+ if (AVIRP == getIRPosition() || !CheckIRP(AVIRP))
+ return indicatePessimisticFixpoint();
+ return ChangeStatus::UNCHANGED;
+ }
+
+ for (const auto &VAC : Values)
+ if (!CheckIRP(IRPosition::value(*VAC.getValue())))
+ return indicatePessimisticFixpoint();
+
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
+};
+
+/// NonNull attribute for function return value.
+struct AANonNullReturned final
+ : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
+ false, AANonNull::IRAttributeKind, false> {
+ AANonNullReturned(const IRPosition &IRP, Attributor &A)
+ : AAReturnedFromReturnedValues<AANonNull, AANonNull, AANonNull::StateType,
+ false, Attribute::NonNull, false>(IRP, A) {
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "nonnull" : "may-null";
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(nonnull) }
+};
+
+/// NonNull attribute for function argument.
+struct AANonNullArgument final
+ : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl> {
+ AANonNullArgument(const IRPosition &IRP, Attributor &A)
+ : AAArgumentFromCallSiteArguments<AANonNull, AANonNullImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nonnull) }
+};
+
+struct AANonNullCallSiteArgument final : AANonNullFloating {
+ AANonNullCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AANonNullFloating(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(nonnull) }
+};
+
+/// NonNull attribute for a call site return position.
+struct AANonNullCallSiteReturned final
+ : AACalleeToCallSite<AANonNull, AANonNullImpl> {
+ AANonNullCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AANonNull, AANonNullImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(nonnull) }
+};
+} // namespace
+
+/// ------------------------ Must-Progress Attributes --------------------------
+namespace {
+struct AAMustProgressImpl : public AAMustProgress {
+ AAMustProgressImpl(const IRPosition &IRP, Attributor &A)
+ : AAMustProgress(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ bool IsKnown;
+ assert(!AA::hasAssumedIRAttr<Attribute::MustProgress>(
+ A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
+ (void)IsKnown;
+ }
+
+ /// See AbstractAttribute::getAsStr()
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "mustprogress" : "may-not-progress";
+ }
+};
+
+struct AAMustProgressFunction final : AAMustProgressImpl {
+ AAMustProgressFunction(const IRPosition &IRP, Attributor &A)
+ : AAMustProgressImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ bool IsKnown;
+ if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
+ A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnown)) {
+ if (IsKnown)
+ return indicateOptimisticFixpoint();
+ return ChangeStatus::UNCHANGED;
+ }
+
+ auto CheckForMustProgress = [&](AbstractCallSite ACS) {
+ IRPosition IPos = IRPosition::callsite_function(*ACS.getInstruction());
+ bool IsKnownMustProgress;
+ return AA::hasAssumedIRAttr<Attribute::MustProgress>(
+ A, this, IPos, DepClassTy::REQUIRED, IsKnownMustProgress,
+ /* IgnoreSubsumingPositions */ true);
+ };
+
+ bool AllCallSitesKnown = true;
+ if (!A.checkForAllCallSites(CheckForMustProgress, *this,
+ /* RequireAllCallSites */ true,
+ AllCallSitesKnown))
+ return indicatePessimisticFixpoint();
+
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FN_ATTR(mustprogress)
+ }
+};
+
+/// MustProgress attribute deduction for a call sites.
+struct AAMustProgressCallSite final : AAMustProgressImpl {
+ AAMustProgressCallSite(const IRPosition &IRP, Attributor &A)
+ : AAMustProgressImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // TODO: Once we have call site specific value information we can provide
+ // call site specific liveness information and then it makes
+ // sense to specialize attributes for call sites arguments instead of
+ // redirecting requests to the callee argument.
+ const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
+ bool IsKnownMustProgress;
+ if (!AA::hasAssumedIRAttr<Attribute::MustProgress>(
+ A, this, FnPos, DepClassTy::REQUIRED, IsKnownMustProgress))
+ return indicatePessimisticFixpoint();
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CS_ATTR(mustprogress);
+ }
+};
+} // namespace
+
+/// ------------------------ No-Recurse Attributes ----------------------------
+
+namespace {
+struct AANoRecurseImpl : public AANoRecurse {
+ AANoRecurseImpl(const IRPosition &IRP, Attributor &A) : AANoRecurse(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ bool IsKnown;
+ assert(!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
+ A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
+ (void)IsKnown;
+ }
+
+ /// See AbstractAttribute::getAsStr()
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "norecurse" : "may-recurse";
+ }
+};
+
+struct AANoRecurseFunction final : AANoRecurseImpl {
+ AANoRecurseFunction(const IRPosition &IRP, Attributor &A)
+ : AANoRecurseImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+
+ // If all live call sites are known to be no-recurse, we are as well.
+ auto CallSitePred = [&](AbstractCallSite ACS) {
+ bool IsKnownNoRecurse;
+ if (!AA::hasAssumedIRAttr<Attribute::NoRecurse>(
+ A, this,
+ IRPosition::function(*ACS.getInstruction()->getFunction()),
+ DepClassTy::NONE, IsKnownNoRecurse))
+ return false;
+ return IsKnownNoRecurse;
+ };
+ bool UsedAssumedInformation = false;
+ if (A.checkForAllCallSites(CallSitePred, *this, true,
+ UsedAssumedInformation)) {
+ // If we know all call sites and all are known no-recurse, we are done.
+ // If all known call sites, which might not be all that exist, are known
+ // to be no-recurse, we are not done but we can continue to assume
+ // no-recurse. If one of the call sites we have not visited will become
+ // live, another update is triggered.
+ if (!UsedAssumedInformation)
+ indicateOptimisticFixpoint();
+ return ChangeStatus::UNCHANGED;
+ }
+
+ const AAInterFnReachability *EdgeReachability =
+ A.getAAFor<AAInterFnReachability>(*this, getIRPosition(),
+ DepClassTy::REQUIRED);
+ if (EdgeReachability && EdgeReachability->canReach(A, *getAnchorScope()))
+ return indicatePessimisticFixpoint();
+ return ChangeStatus::UNCHANGED;
+ }
+
+ void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
+};
+
+/// NoRecurse attribute deduction for a call sites.
+struct AANoRecurseCallSite final
+ : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl> {
+ AANoRecurseCallSite(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AANoRecurse, AANoRecurseImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(norecurse); }
+};
+} // namespace
+
+/// ------------------------ No-Convergent Attribute --------------------------
+
+namespace {
+struct AANonConvergentImpl : public AANonConvergent {
+ AANonConvergentImpl(const IRPosition &IRP, Attributor &A)
+ : AANonConvergent(IRP, A) {}
+
+ /// See AbstractAttribute::getAsStr()
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "non-convergent" : "may-be-convergent";
+ }
+};
+
+struct AANonConvergentFunction final : AANonConvergentImpl {
+ AANonConvergentFunction(const IRPosition &IRP, Attributor &A)
+ : AANonConvergentImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // If all function calls are known to not be convergent, we are not
+ // convergent.
+ auto CalleeIsNotConvergent = [&](Instruction &Inst) {
+ CallBase &CB = cast<CallBase>(Inst);
+ auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand());
+ if (!Callee || Callee->isIntrinsic()) {
+ return false;
+ }
+ if (Callee->isDeclaration()) {
+ return !Callee->hasFnAttribute(Attribute::Convergent);
+ }
+ const auto *ConvergentAA = A.getAAFor<AANonConvergent>(
+ *this, IRPosition::function(*Callee), DepClassTy::REQUIRED);
+ return ConvergentAA && ConvergentAA->isAssumedNotConvergent();
+ };
+
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllCallLikeInstructions(CalleeIsNotConvergent, *this,
+ UsedAssumedInformation)) {
+ return indicatePessimisticFixpoint();
+ }
+ return ChangeStatus::UNCHANGED;
+ }
+
+ ChangeStatus manifest(Attributor &A) override {
+ if (isKnownNotConvergent() &&
+ A.hasAttr(getIRPosition(), Attribute::Convergent)) {
+ A.removeAttrs(getIRPosition(), {Attribute::Convergent});
+ return ChangeStatus::CHANGED;
+ }
+ return ChangeStatus::UNCHANGED;
+ }
+
+ void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(convergent) }
+};
+} // namespace
+
+/// -------------------- Undefined-Behavior Attributes ------------------------
+
+namespace {
+struct AAUndefinedBehaviorImpl : public AAUndefinedBehavior {
+ AAUndefinedBehaviorImpl(const IRPosition &IRP, Attributor &A)
+ : AAUndefinedBehavior(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ // through a pointer (i.e. also branches etc.)
+ ChangeStatus updateImpl(Attributor &A) override {
+ const size_t UBPrevSize = KnownUBInsts.size();
+ const size_t NoUBPrevSize = AssumedNoUBInsts.size();
+
+ auto InspectMemAccessInstForUB = [&](Instruction &I) {
+ // Lang ref now states volatile store is not UB, let's skip them.
+ if (I.isVolatile() && I.mayWriteToMemory())
+ return true;
+
+ // Skip instructions that are already saved.
+ if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
+ return true;
+
+ // If we reach here, we know we have an instruction
+ // that accesses memory through a pointer operand,
+ // for which getPointerOperand() should give it to us.
+ Value *PtrOp =
+ const_cast<Value *>(getPointerOperand(&I, /* AllowVolatile */ true));
+ assert(PtrOp &&
+ "Expected pointer operand of memory accessing instruction");
+
+ // Either we stopped and the appropriate action was taken,
+ // or we got back a simplified value to continue.
+ std::optional<Value *> SimplifiedPtrOp =
+ stopOnUndefOrAssumed(A, PtrOp, &I);
+ if (!SimplifiedPtrOp || !*SimplifiedPtrOp)
+ return true;
+ const Value *PtrOpVal = *SimplifiedPtrOp;
+
+ // A memory access through a pointer is considered UB
+ // only if the pointer has constant null value.
+ // TODO: Expand it to not only check constant values.
+ if (!isa<ConstantPointerNull>(PtrOpVal)) {
+ AssumedNoUBInsts.insert(&I);
+ return true;
+ }
+ const Type *PtrTy = PtrOpVal->getType();
+
+ // Because we only consider instructions inside functions,
+ // assume that a parent function exists.
+ const Function *F = I.getFunction();
+
+ // A memory access using constant null pointer is only considered UB
+ // if null pointer is _not_ defined for the target platform.
+ if (llvm::NullPointerIsDefined(F, PtrTy->getPointerAddressSpace()))
+ AssumedNoUBInsts.insert(&I);
+ else
+ KnownUBInsts.insert(&I);
+ return true;
+ };
+
+ auto InspectBrInstForUB = [&](Instruction &I) {
+ // A conditional branch instruction is considered UB if it has `undef`
+ // condition.
+
+ // Skip instructions that are already saved.
+ if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
+ return true;
+
+ // We know we have a branch instruction.
+ auto *BrInst = cast<BranchInst>(&I);
+
+ // Unconditional branches are never considered UB.
+ if (BrInst->isUnconditional())
+ return true;
+
+ // Either we stopped and the appropriate action was taken,
+ // or we got back a simplified value to continue.
+ std::optional<Value *> SimplifiedCond =
+ stopOnUndefOrAssumed(A, BrInst->getCondition(), BrInst);
+ if (!SimplifiedCond || !*SimplifiedCond)
+ return true;
+ AssumedNoUBInsts.insert(&I);
+ return true;
+ };
+
+ auto InspectCallSiteForUB = [&](Instruction &I) {
+ // Check whether a callsite always cause UB or not
+
+ // Skip instructions that are already saved.
+ if (AssumedNoUBInsts.count(&I) || KnownUBInsts.count(&I))
+ return true;
+
+ // Check nonnull and noundef argument attribute violation for each
+ // callsite.
+ CallBase &CB = cast<CallBase>(I);
+ auto *Callee = dyn_cast_if_present<Function>(CB.getCalledOperand());
+ if (!Callee)
+ return true;
+ for (unsigned idx = 0; idx < CB.arg_size(); idx++) {
+ // If current argument is known to be simplified to null pointer and the
+ // corresponding argument position is known to have nonnull attribute,
+ // the argument is poison. Furthermore, if the argument is poison and
+ // the position is known to have noundef attriubte, this callsite is
+ // considered UB.
+ if (idx >= Callee->arg_size())
+ break;
+ Value *ArgVal = CB.getArgOperand(idx);
+ if (!ArgVal)
+ continue;
+ // Here, we handle three cases.
+ // (1) Not having a value means it is dead. (we can replace the value
+ // with undef)
+ // (2) Simplified to undef. The argument violate noundef attriubte.
+ // (3) Simplified to null pointer where known to be nonnull.
+ // The argument is a poison value and violate noundef attribute.
+ IRPosition CalleeArgumentIRP = IRPosition::callsite_argument(CB, idx);
+ bool IsKnownNoUndef;
+ AA::hasAssumedIRAttr<Attribute::NoUndef>(
+ A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNoUndef);
+ if (!IsKnownNoUndef)
+ continue;
+ bool UsedAssumedInformation = false;
+ std::optional<Value *> SimplifiedVal =
+ A.getAssumedSimplified(IRPosition::value(*ArgVal), *this,
+ UsedAssumedInformation, AA::Interprocedural);
+ if (UsedAssumedInformation)
+ continue;
+ if (SimplifiedVal && !*SimplifiedVal)
+ return true;
+ if (!SimplifiedVal || isa<UndefValue>(**SimplifiedVal)) {
+ KnownUBInsts.insert(&I);
+ continue;
+ }
+ if (!ArgVal->getType()->isPointerTy() ||
+ !isa<ConstantPointerNull>(**SimplifiedVal))
+ continue;
+ bool IsKnownNonNull;
+ AA::hasAssumedIRAttr<Attribute::NonNull>(
+ A, this, CalleeArgumentIRP, DepClassTy::NONE, IsKnownNonNull);
+ if (IsKnownNonNull)
+ KnownUBInsts.insert(&I);
+ }
+ return true;
+ };
+
+ auto InspectReturnInstForUB = [&](Instruction &I) {
+ auto &RI = cast<ReturnInst>(I);
+ // Either we stopped and the appropriate action was taken,
+ // or we got back a simplified return value to continue.
+ std::optional<Value *> SimplifiedRetValue =
+ stopOnUndefOrAssumed(A, RI.getReturnValue(), &I);
+ if (!SimplifiedRetValue || !*SimplifiedRetValue)
+ return true;
+
+ // Check if a return instruction always cause UB or not
+ // Note: It is guaranteed that the returned position of the anchor
+ // scope has noundef attribute when this is called.
+ // We also ensure the return position is not "assumed dead"
+ // because the returned value was then potentially simplified to
+ // `undef` in AAReturnedValues without removing the `noundef`
+ // attribute yet.
+
+ // When the returned position has noundef attriubte, UB occurs in the
+ // following cases.
+ // (1) Returned value is known to be undef.
+ // (2) The value is known to be a null pointer and the returned
+ // position has nonnull attribute (because the returned value is
+ // poison).
+ if (isa<ConstantPointerNull>(*SimplifiedRetValue)) {
+ bool IsKnownNonNull;
+ AA::hasAssumedIRAttr<Attribute::NonNull>(
+ A, this, IRPosition::returned(*getAnchorScope()), DepClassTy::NONE,
+ IsKnownNonNull);
+ if (IsKnownNonNull)
+ KnownUBInsts.insert(&I);
+ }
+
+ return true;
+ };
+
+ bool UsedAssumedInformation = false;
+ A.checkForAllInstructions(InspectMemAccessInstForUB, *this,
+ {Instruction::Load, Instruction::Store,
+ Instruction::AtomicCmpXchg,
+ Instruction::AtomicRMW},
+ UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ true);
+ A.checkForAllInstructions(InspectBrInstForUB, *this, {Instruction::Br},
+ UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ true);
+ A.checkForAllCallLikeInstructions(InspectCallSiteForUB, *this,
+ UsedAssumedInformation);
+
+ // If the returned position of the anchor scope has noundef attriubte, check
+ // all returned instructions.
+ if (!getAnchorScope()->getReturnType()->isVoidTy()) {
+ const IRPosition &ReturnIRP = IRPosition::returned(*getAnchorScope());
+ if (!A.isAssumedDead(ReturnIRP, this, nullptr, UsedAssumedInformation)) {
+ bool IsKnownNoUndef;
+ AA::hasAssumedIRAttr<Attribute::NoUndef>(
+ A, this, ReturnIRP, DepClassTy::NONE, IsKnownNoUndef);
+ if (IsKnownNoUndef)
+ A.checkForAllInstructions(InspectReturnInstForUB, *this,
+ {Instruction::Ret}, UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ true);
+ }
+ }
+
+ if (NoUBPrevSize != AssumedNoUBInsts.size() ||
+ UBPrevSize != KnownUBInsts.size())
+ return ChangeStatus::CHANGED;
+ return ChangeStatus::UNCHANGED;
+ }
+
+ bool isKnownToCauseUB(Instruction *I) const override {
+ return KnownUBInsts.count(I);
+ }
+
+ bool isAssumedToCauseUB(Instruction *I) const override {
+ // In simple words, if an instruction is not in the assumed to _not_
+ // cause UB, then it is assumed UB (that includes those
+ // in the KnownUBInsts set). The rest is boilerplate
+ // is to ensure that it is one of the instructions we test
+ // for UB.
+
+ switch (I->getOpcode()) {
+ case Instruction::Load:
+ case Instruction::Store:
+ case Instruction::AtomicCmpXchg:
+ case Instruction::AtomicRMW:
+ return !AssumedNoUBInsts.count(I);
+ case Instruction::Br: {
+ auto *BrInst = cast<BranchInst>(I);
+ if (BrInst->isUnconditional())
+ return false;
+ return !AssumedNoUBInsts.count(I);
+ } break;
+ default:
+ return false;
+ }
+ return false;
+ }
+
+ ChangeStatus manifest(Attributor &A) override {
+ if (KnownUBInsts.empty())
+ return ChangeStatus::UNCHANGED;
+ for (Instruction *I : KnownUBInsts)
+ A.changeToUnreachableAfterManifest(I);
+ return ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractAttribute::getAsStr()
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "undefined-behavior" : "no-ub";
+ }
+
+ /// Note: The correctness of this analysis depends on the fact that the
+ /// following 2 sets will stop changing after some point.
+ /// "Change" here means that their size changes.
+ /// The size of each set is monotonically increasing
+ /// (we only add items to them) and it is upper bounded by the number of
+ /// instructions in the processed function (we can never save more
+ /// elements in either set than this number). Hence, at some point,
+ /// they will stop increasing.
+ /// Consequently, at some point, both sets will have stopped
+ /// changing, effectively making the analysis reach a fixpoint.
+
+ /// Note: These 2 sets are disjoint and an instruction can be considered
+ /// one of 3 things:
+ /// 1) Known to cause UB (AAUndefinedBehavior could prove it) and put it in
+ /// the KnownUBInsts set.
+ /// 2) Assumed to cause UB (in every updateImpl, AAUndefinedBehavior
+ /// has a reason to assume it).
+ /// 3) Assumed to not cause UB. very other instruction - AAUndefinedBehavior
+ /// could not find a reason to assume or prove that it can cause UB,
+ /// hence it assumes it doesn't. We have a set for these instructions
+ /// so that we don't reprocess them in every update.
+ /// Note however that instructions in this set may cause UB.
+
+protected:
+ /// A set of all live instructions _known_ to cause UB.
+ SmallPtrSet<Instruction *, 8> KnownUBInsts;
+
+private:
+ /// A set of all the (live) instructions that are assumed to _not_ cause UB.
+ SmallPtrSet<Instruction *, 8> AssumedNoUBInsts;
+
+ // Should be called on updates in which if we're processing an instruction
+ // \p I that depends on a value \p V, one of the following has to happen:
+ // - If the value is assumed, then stop.
+ // - If the value is known but undef, then consider it UB.
+ // - Otherwise, do specific processing with the simplified value.
+ // We return std::nullopt in the first 2 cases to signify that an appropriate
+ // action was taken and the caller should stop.
+ // Otherwise, we return the simplified value that the caller should
+ // use for specific processing.
+ std::optional<Value *> stopOnUndefOrAssumed(Attributor &A, Value *V,
+ Instruction *I) {
+ bool UsedAssumedInformation = false;
+ std::optional<Value *> SimplifiedV =
+ A.getAssumedSimplified(IRPosition::value(*V), *this,
+ UsedAssumedInformation, AA::Interprocedural);
+ if (!UsedAssumedInformation) {
+ // Don't depend on assumed values.
+ if (!SimplifiedV) {
+ // If it is known (which we tested above) but it doesn't have a value,
+ // then we can assume `undef` and hence the instruction is UB.
+ KnownUBInsts.insert(I);
+ return std::nullopt;
+ }
+ if (!*SimplifiedV)
+ return nullptr;
+ V = *SimplifiedV;
+ }
+ if (isa<UndefValue>(V)) {
+ KnownUBInsts.insert(I);
+ return std::nullopt;
+ }
+ return V;
+ }
+};
+
+struct AAUndefinedBehaviorFunction final : AAUndefinedBehaviorImpl {
+ AAUndefinedBehaviorFunction(const IRPosition &IRP, Attributor &A)
+ : AAUndefinedBehaviorImpl(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECL(UndefinedBehaviorInstruction, Instruction,
+ "Number of instructions known to have UB");
+ BUILD_STAT_NAME(UndefinedBehaviorInstruction, Instruction) +=
+ KnownUBInsts.size();
+ }
+};
+} // namespace
+
+/// ------------------------ Will-Return Attributes ----------------------------
+
+namespace {
+// Helper function that checks whether a function has any cycle which we don't
+// know if it is bounded or not.
+// Loops with maximum trip count are considered bounded, any other cycle not.
+static bool mayContainUnboundedCycle(Function &F, Attributor &A) {
+ ScalarEvolution *SE =
+ A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(F);
+ LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(F);
+ // If either SCEV or LoopInfo is not available for the function then we assume
+ // any cycle to be unbounded cycle.
+ // We use scc_iterator which uses Tarjan algorithm to find all the maximal
+ // SCCs.To detect if there's a cycle, we only need to find the maximal ones.
+ if (!SE || !LI) {
+ for (scc_iterator<Function *> SCCI = scc_begin(&F); !SCCI.isAtEnd(); ++SCCI)
+ if (SCCI.hasCycle())
+ return true;
+ return false;
+ }
+
+ // If there's irreducible control, the function may contain non-loop cycles.
+ if (mayContainIrreducibleControl(F, LI))
+ return true;
+
+ // Any loop that does not have a max trip count is considered unbounded cycle.
+ for (auto *L : LI->getLoopsInPreorder()) {
+ if (!SE->getSmallConstantMaxTripCount(L))
+ return true;
+ }
+ return false;
+}
+
+struct AAWillReturnImpl : public AAWillReturn {
+ AAWillReturnImpl(const IRPosition &IRP, Attributor &A)
+ : AAWillReturn(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ bool IsKnown;
+ assert(!AA::hasAssumedIRAttr<Attribute::WillReturn>(
+ A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
+ (void)IsKnown;
+ }
+
+ /// Check for `mustprogress` and `readonly` as they imply `willreturn`.
+ bool isImpliedByMustprogressAndReadonly(Attributor &A, bool KnownOnly) {
+ if (!A.hasAttr(getIRPosition(), {Attribute::MustProgress}))
+ return false;
+
+ bool IsKnown;
+ if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
+ return IsKnown || !KnownOnly;
+ return false;
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
+ return ChangeStatus::UNCHANGED;
+
+ auto CheckForWillReturn = [&](Instruction &I) {
+ IRPosition IPos = IRPosition::callsite_function(cast<CallBase>(I));
+ bool IsKnown;
+ if (AA::hasAssumedIRAttr<Attribute::WillReturn>(
+ A, this, IPos, DepClassTy::REQUIRED, IsKnown)) {
+ if (IsKnown)
+ return true;
+ } else {
+ return false;
+ }
+ bool IsKnownNoRecurse;
+ return AA::hasAssumedIRAttr<Attribute::NoRecurse>(
+ A, this, IPos, DepClassTy::REQUIRED, IsKnownNoRecurse);
+ };
+
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllCallLikeInstructions(CheckForWillReturn, *this,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::getAsStr()
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "willreturn" : "may-noreturn";
+ }
+};
+
+struct AAWillReturnFunction final : AAWillReturnImpl {
+ AAWillReturnFunction(const IRPosition &IRP, Attributor &A)
+ : AAWillReturnImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ AAWillReturnImpl::initialize(A);
+
+ Function *F = getAnchorScope();
+ assert(F && "Did expect an anchor function");
+ if (F->isDeclaration() || mayContainUnboundedCycle(*F, A))
+ indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(willreturn) }
+};
+
+/// WillReturn attribute deduction for a call sites.
+struct AAWillReturnCallSite final
+ : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl> {
+ AAWillReturnCallSite(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AAWillReturn, AAWillReturnImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ if (isImpliedByMustprogressAndReadonly(A, /* KnownOnly */ false))
+ return ChangeStatus::UNCHANGED;
+
+ return AACalleeToCallSite::updateImpl(A);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(willreturn); }
+};
+} // namespace
+
+/// -------------------AAIntraFnReachability Attribute--------------------------
+
+/// All information associated with a reachability query. This boilerplate code
+/// is used by both AAIntraFnReachability and AAInterFnReachability, with
+/// different \p ToTy values.
+template <typename ToTy> struct ReachabilityQueryInfo {
+ enum class Reachable {
+ No,
+ Yes,
+ };
+
+ /// Start here,
+ const Instruction *From = nullptr;
+ /// reach this place,
+ const ToTy *To = nullptr;
+ /// without going through any of these instructions,
+ const AA::InstExclusionSetTy *ExclusionSet = nullptr;
+ /// and remember if it worked:
+ Reachable Result = Reachable::No;
+
+ /// Precomputed hash for this RQI.
+ unsigned Hash = 0;
+
+ unsigned computeHashValue() const {
+ assert(Hash == 0 && "Computed hash twice!");
+ using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
+ using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
+ return const_cast<ReachabilityQueryInfo<ToTy> *>(this)->Hash =
+ detail::combineHashValue(PairDMI ::getHashValue({From, To}),
+ InstSetDMI::getHashValue(ExclusionSet));
+ }
+
+ ReachabilityQueryInfo(const Instruction *From, const ToTy *To)
+ : From(From), To(To) {}
+
+ /// Constructor replacement to ensure unique and stable sets are used for the
+ /// cache.
+ ReachabilityQueryInfo(Attributor &A, const Instruction &From, const ToTy &To,
+ const AA::InstExclusionSetTy *ES, bool MakeUnique)
+ : From(&From), To(&To), ExclusionSet(ES) {
+
+ if (!ES || ES->empty()) {
+ ExclusionSet = nullptr;
+ } else if (MakeUnique) {
+ ExclusionSet = A.getInfoCache().getOrCreateUniqueBlockExecutionSet(ES);
+ }
+ }
+
+ ReachabilityQueryInfo(const ReachabilityQueryInfo &RQI)
+ : From(RQI.From), To(RQI.To), ExclusionSet(RQI.ExclusionSet) {}
+};
+
+namespace llvm {
+template <typename ToTy> struct DenseMapInfo<ReachabilityQueryInfo<ToTy> *> {
+ using InstSetDMI = DenseMapInfo<const AA::InstExclusionSetTy *>;
+ using PairDMI = DenseMapInfo<std::pair<const Instruction *, const ToTy *>>;
+
+ static ReachabilityQueryInfo<ToTy> EmptyKey;
+ static ReachabilityQueryInfo<ToTy> TombstoneKey;
+
+ static inline ReachabilityQueryInfo<ToTy> *getEmptyKey() { return &EmptyKey; }
+ static inline ReachabilityQueryInfo<ToTy> *getTombstoneKey() {
+ return &TombstoneKey;
+ }
+ static unsigned getHashValue(const ReachabilityQueryInfo<ToTy> *RQI) {
+ return RQI->Hash ? RQI->Hash : RQI->computeHashValue();
+ }
+ static bool isEqual(const ReachabilityQueryInfo<ToTy> *LHS,
+ const ReachabilityQueryInfo<ToTy> *RHS) {
+ if (!PairDMI::isEqual({LHS->From, LHS->To}, {RHS->From, RHS->To}))
+ return false;
+ return InstSetDMI::isEqual(LHS->ExclusionSet, RHS->ExclusionSet);
+ }
+};
+
+#define DefineKeys(ToTy) \
+ template <> \
+ ReachabilityQueryInfo<ToTy> \
+ DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::EmptyKey = \
+ ReachabilityQueryInfo<ToTy>( \
+ DenseMapInfo<const Instruction *>::getEmptyKey(), \
+ DenseMapInfo<const ToTy *>::getEmptyKey()); \
+ template <> \
+ ReachabilityQueryInfo<ToTy> \
+ DenseMapInfo<ReachabilityQueryInfo<ToTy> *>::TombstoneKey = \
+ ReachabilityQueryInfo<ToTy>( \
+ DenseMapInfo<const Instruction *>::getTombstoneKey(), \
+ DenseMapInfo<const ToTy *>::getTombstoneKey());
+
+DefineKeys(Instruction) DefineKeys(Function)
+#undef DefineKeys
+
+} // namespace llvm
+
+namespace {
+
+template <typename BaseTy, typename ToTy>
+struct CachedReachabilityAA : public BaseTy {
+ using RQITy = ReachabilityQueryInfo<ToTy>;
+
+ CachedReachabilityAA(const IRPosition &IRP, Attributor &A) : BaseTy(IRP, A) {}
+
+ /// See AbstractAttribute::isQueryAA.
+ bool isQueryAA() const override { return true; }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ for (unsigned u = 0, e = QueryVector.size(); u < e; ++u) {
+ RQITy *RQI = QueryVector[u];
+ if (RQI->Result == RQITy::Reachable::No &&
+ isReachableImpl(A, *RQI, /*IsTemporaryRQI=*/false))
+ Changed = ChangeStatus::CHANGED;
+ }
+ return Changed;
+ }
+
+ virtual bool isReachableImpl(Attributor &A, RQITy &RQI,
+ bool IsTemporaryRQI) = 0;
+
+ bool rememberResult(Attributor &A, typename RQITy::Reachable Result,
+ RQITy &RQI, bool UsedExclusionSet, bool IsTemporaryRQI) {
+ RQI.Result = Result;
+
+ // Remove the temporary RQI from the cache.
+ if (IsTemporaryRQI)
+ QueryCache.erase(&RQI);
+
+ // Insert a plain RQI (w/o exclusion set) if that makes sense. Two options:
+ // 1) If it is reachable, it doesn't matter if we have an exclusion set for
+ // this query. 2) We did not use the exclusion set, potentially because
+ // there is none.
+ if (Result == RQITy::Reachable::Yes || !UsedExclusionSet) {
+ RQITy PlainRQI(RQI.From, RQI.To);
+ if (!QueryCache.count(&PlainRQI)) {
+ RQITy *RQIPtr = new (A.Allocator) RQITy(RQI.From, RQI.To);
+ RQIPtr->Result = Result;
+ QueryVector.push_back(RQIPtr);
+ QueryCache.insert(RQIPtr);
+ }
+ }
+
+ // Check if we need to insert a new permanent RQI with the exclusion set.
+ if (IsTemporaryRQI && Result != RQITy::Reachable::Yes && UsedExclusionSet) {
+ assert((!RQI.ExclusionSet || !RQI.ExclusionSet->empty()) &&
+ "Did not expect empty set!");
+ RQITy *RQIPtr = new (A.Allocator)
+ RQITy(A, *RQI.From, *RQI.To, RQI.ExclusionSet, true);
+ assert(RQIPtr->Result == RQITy::Reachable::No && "Already reachable?");
+ RQIPtr->Result = Result;
+ assert(!QueryCache.count(RQIPtr));
+ QueryVector.push_back(RQIPtr);
+ QueryCache.insert(RQIPtr);
+ }
+
+ if (Result == RQITy::Reachable::No && IsTemporaryRQI)
+ A.registerForUpdate(*this);
+ return Result == RQITy::Reachable::Yes;
+ }
+
+ const std::string getAsStr(Attributor *A) const override {
+ // TODO: Return the number of reachable queries.
+ return "#queries(" + std::to_string(QueryVector.size()) + ")";
+ }
+
+ bool checkQueryCache(Attributor &A, RQITy &StackRQI,
+ typename RQITy::Reachable &Result) {
+ if (!this->getState().isValidState()) {
+ Result = RQITy::Reachable::Yes;
+ return true;
+ }
+
+ // If we have an exclusion set we might be able to find our answer by
+ // ignoring it first.
+ if (StackRQI.ExclusionSet) {
+ RQITy PlainRQI(StackRQI.From, StackRQI.To);
+ auto It = QueryCache.find(&PlainRQI);
+ if (It != QueryCache.end() && (*It)->Result == RQITy::Reachable::No) {
+ Result = RQITy::Reachable::No;
+ return true;
+ }
+ }
+
+ auto It = QueryCache.find(&StackRQI);
+ if (It != QueryCache.end()) {
+ Result = (*It)->Result;
+ return true;
+ }
+
+ // Insert a temporary for recursive queries. We will replace it with a
+ // permanent entry later.
+ QueryCache.insert(&StackRQI);
+ return false;
+ }
+
+private:
+ SmallVector<RQITy *> QueryVector;
+ DenseSet<RQITy *> QueryCache;
+};
+
+struct AAIntraFnReachabilityFunction final
+ : public CachedReachabilityAA<AAIntraFnReachability, Instruction> {
+ using Base = CachedReachabilityAA<AAIntraFnReachability, Instruction>;
+ AAIntraFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {
+ DT = A.getInfoCache().getAnalysisResultForFunction<DominatorTreeAnalysis>(
+ *IRP.getAssociatedFunction());
+ }
+
+ bool isAssumedReachable(
+ Attributor &A, const Instruction &From, const Instruction &To,
+ const AA::InstExclusionSetTy *ExclusionSet) const override {
+ auto *NonConstThis = const_cast<AAIntraFnReachabilityFunction *>(this);
+ if (&From == &To)
+ return true;
+
+ RQITy StackRQI(A, From, To, ExclusionSet, false);
+ typename RQITy::Reachable Result;
+ if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
+ return NonConstThis->isReachableImpl(A, StackRQI,
+ /*IsTemporaryRQI=*/true);
+ return Result == RQITy::Reachable::Yes;
+ }
+
+ ChangeStatus updateImpl(Attributor &A) override {
+ // We only depend on liveness. DeadEdges is all we care about, check if any
+ // of them changed.
+ auto *LivenessAA =
+ A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
+ if (LivenessAA &&
+ llvm::all_of(DeadEdges,
+ [&](const auto &DeadEdge) {
+ return LivenessAA->isEdgeDead(DeadEdge.first,
+ DeadEdge.second);
+ }) &&
+ llvm::all_of(DeadBlocks, [&](const BasicBlock *BB) {
+ return LivenessAA->isAssumedDead(BB);
+ })) {
+ return ChangeStatus::UNCHANGED;
+ }
+ DeadEdges.clear();
+ DeadBlocks.clear();
+ return Base::updateImpl(A);
+ }
+
+ bool isReachableImpl(Attributor &A, RQITy &RQI,
+ bool IsTemporaryRQI) override {
+ const Instruction *Origin = RQI.From;
+ bool UsedExclusionSet = false;
+
+ auto WillReachInBlock = [&](const Instruction &From, const Instruction &To,
+ const AA::InstExclusionSetTy *ExclusionSet) {
+ const Instruction *IP = &From;
+ while (IP && IP != &To) {
+ if (ExclusionSet && IP != Origin && ExclusionSet->count(IP)) {
+ UsedExclusionSet = true;
+ break;
+ }
+ IP = IP->getNextNode();
+ }
+ return IP == &To;
+ };
+
+ const BasicBlock *FromBB = RQI.From->getParent();
+ const BasicBlock *ToBB = RQI.To->getParent();
+ assert(FromBB->getParent() == ToBB->getParent() &&
+ "Not an intra-procedural query!");
+
+ // Check intra-block reachability, however, other reaching paths are still
+ // possible.
+ if (FromBB == ToBB &&
+ WillReachInBlock(*RQI.From, *RQI.To, RQI.ExclusionSet))
+ return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
+ IsTemporaryRQI);
+
+ // Check if reaching the ToBB block is sufficient or if even that would not
+ // ensure reaching the target. In the latter case we are done.
+ if (!WillReachInBlock(ToBB->front(), *RQI.To, RQI.ExclusionSet))
+ return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
+ IsTemporaryRQI);
+
+ const Function *Fn = FromBB->getParent();
+ SmallPtrSet<const BasicBlock *, 16> ExclusionBlocks;
+ if (RQI.ExclusionSet)
+ for (auto *I : *RQI.ExclusionSet)
+ if (I->getFunction() == Fn)
+ ExclusionBlocks.insert(I->getParent());
+
+ // Check if we make it out of the FromBB block at all.
+ if (ExclusionBlocks.count(FromBB) &&
+ !WillReachInBlock(*RQI.From, *FromBB->getTerminator(),
+ RQI.ExclusionSet))
+ return rememberResult(A, RQITy::Reachable::No, RQI, true, IsTemporaryRQI);
+
+ auto *LivenessAA =
+ A.getAAFor<AAIsDead>(*this, getIRPosition(), DepClassTy::OPTIONAL);
+ if (LivenessAA && LivenessAA->isAssumedDead(ToBB)) {
+ DeadBlocks.insert(ToBB);
+ return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
+ IsTemporaryRQI);
+ }
+
+ SmallPtrSet<const BasicBlock *, 16> Visited;
+ SmallVector<const BasicBlock *, 16> Worklist;
+ Worklist.push_back(FromBB);
+
+ DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> LocalDeadEdges;
+ while (!Worklist.empty()) {
+ const BasicBlock *BB = Worklist.pop_back_val();
+ if (!Visited.insert(BB).second)
+ continue;
+ for (const BasicBlock *SuccBB : successors(BB)) {
+ if (LivenessAA && LivenessAA->isEdgeDead(BB, SuccBB)) {
+ LocalDeadEdges.insert({BB, SuccBB});
+ continue;
+ }
+ // We checked before if we just need to reach the ToBB block.
+ if (SuccBB == ToBB)
+ return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
+ IsTemporaryRQI);
+ if (DT && ExclusionBlocks.empty() && DT->dominates(BB, ToBB))
+ return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
+ IsTemporaryRQI);
+
+ if (ExclusionBlocks.count(SuccBB)) {
+ UsedExclusionSet = true;
+ continue;
+ }
+ Worklist.push_back(SuccBB);
+ }
+ }
+
+ DeadEdges.insert(LocalDeadEdges.begin(), LocalDeadEdges.end());
+ return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
+ IsTemporaryRQI);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {}
+
+private:
+ // Set of assumed dead blocks we used in the last query. If any changes we
+ // update the state.
+ DenseSet<const BasicBlock *> DeadBlocks;
+
+ // Set of assumed dead edges we used in the last query. If any changes we
+ // update the state.
+ DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> DeadEdges;
+
+ /// The dominator tree of the function to short-circuit reasoning.
+ const DominatorTree *DT = nullptr;
+};
+} // namespace
+
+/// ------------------------ NoAlias Argument Attribute ------------------------
+
+bool AANoAlias::isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions) {
+ assert(ImpliedAttributeKind == Attribute::NoAlias &&
+ "Unexpected attribute kind");
+ Value *Val = &IRP.getAssociatedValue();
+ if (IRP.getPositionKind() != IRP_CALL_SITE_ARGUMENT) {
+ if (isa<AllocaInst>(Val))
+ return true;
+ } else {
+ IgnoreSubsumingPositions = true;
+ }
+
+ if (isa<UndefValue>(Val))
+ return true;
+
+ if (isa<ConstantPointerNull>(Val) &&
+ !NullPointerIsDefined(IRP.getAnchorScope(),
+ Val->getType()->getPointerAddressSpace()))
+ return true;
+
+ if (A.hasAttr(IRP, {Attribute::ByVal, Attribute::NoAlias},
+ IgnoreSubsumingPositions, Attribute::NoAlias))
+ return true;
+
+ return false;
+}
+
+namespace {
+struct AANoAliasImpl : AANoAlias {
+ AANoAliasImpl(const IRPosition &IRP, Attributor &A) : AANoAlias(IRP, A) {
+ assert(getAssociatedType()->isPointerTy() &&
+ "Noalias is a pointer attribute");
+ }
+
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "noalias" : "may-alias";
+ }
+};
+
+/// NoAlias attribute for a floating value.
+struct AANoAliasFloating final : AANoAliasImpl {
+ AANoAliasFloating(const IRPosition &IRP, Attributor &A)
+ : AANoAliasImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // TODO: Implement this.
+ return indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(noalias)
+ }
+};
+
+/// NoAlias attribute for an argument.
+struct AANoAliasArgument final
+ : AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl> {
+ using Base = AAArgumentFromCallSiteArguments<AANoAlias, AANoAliasImpl>;
+ AANoAliasArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
+
+ /// See AbstractAttribute::update(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // We have to make sure no-alias on the argument does not break
+ // synchronization when this is a callback argument, see also [1] below.
+ // If synchronization cannot be affected, we delegate to the base updateImpl
+ // function, otherwise we give up for now.
+
+ // If the function is no-sync, no-alias cannot break synchronization.
+ bool IsKnownNoSycn;
+ if (AA::hasAssumedIRAttr<Attribute::NoSync>(
+ A, this, IRPosition::function_scope(getIRPosition()),
+ DepClassTy::OPTIONAL, IsKnownNoSycn))
+ return Base::updateImpl(A);
+
+ // If the argument is read-only, no-alias cannot break synchronization.
+ bool IsKnown;
+ if (AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
+ return Base::updateImpl(A);
+
+ // If the argument is never passed through callbacks, no-alias cannot break
+ // synchronization.
+ bool UsedAssumedInformation = false;
+ if (A.checkForAllCallSites(
+ [](AbstractCallSite ACS) { return !ACS.isCallbackCall(); }, *this,
+ true, UsedAssumedInformation))
+ return Base::updateImpl(A);
+
+ // TODO: add no-alias but make sure it doesn't break synchronization by
+ // introducing fake uses. See:
+ // [1] Compiler Optimizations for OpenMP, J. Doerfert and H. Finkel,
+ // International Workshop on OpenMP 2018,
+ // http://compilers.cs.uni-saarland.de/people/doerfert/par_opt18.pdf
+
+ return indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noalias) }
+};
+
+struct AANoAliasCallSiteArgument final : AANoAliasImpl {
+ AANoAliasCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AANoAliasImpl(IRP, A) {}
+
+ /// Determine if the underlying value may alias with the call site argument
+ /// \p OtherArgNo of \p ICS (= the underlying call site).
+ bool mayAliasWithArgument(Attributor &A, AAResults *&AAR,
+ const AAMemoryBehavior &MemBehaviorAA,
+ const CallBase &CB, unsigned OtherArgNo) {
+ // We do not need to worry about aliasing with the underlying IRP.
+ if (this->getCalleeArgNo() == (int)OtherArgNo)
+ return false;
+
+ // If it is not a pointer or pointer vector we do not alias.
+ const Value *ArgOp = CB.getArgOperand(OtherArgNo);
+ if (!ArgOp->getType()->isPtrOrPtrVectorTy())
+ return false;
+
+ auto *CBArgMemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
+ *this, IRPosition::callsite_argument(CB, OtherArgNo), DepClassTy::NONE);
+
+ // If the argument is readnone, there is no read-write aliasing.
+ if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadNone()) {
+ A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);
+ return false;
+ }
+
+ // If the argument is readonly and the underlying value is readonly, there
+ // is no read-write aliasing.
+ bool IsReadOnly = MemBehaviorAA.isAssumedReadOnly();
+ if (CBArgMemBehaviorAA && CBArgMemBehaviorAA->isAssumedReadOnly() &&
+ IsReadOnly) {
+ A.recordDependence(MemBehaviorAA, *this, DepClassTy::OPTIONAL);
+ A.recordDependence(*CBArgMemBehaviorAA, *this, DepClassTy::OPTIONAL);
+ return false;
+ }
+
+ // We have to utilize actual alias analysis queries so we need the object.
+ if (!AAR)
+ AAR = A.getInfoCache().getAnalysisResultForFunction<AAManager>(
+ *getAnchorScope());
+
+ // Try to rule it out at the call site.
+ bool IsAliasing = !AAR || !AAR->isNoAlias(&getAssociatedValue(), ArgOp);
+ LLVM_DEBUG(dbgs() << "[NoAliasCSArg] Check alias between "
+ "callsite arguments: "
+ << getAssociatedValue() << " " << *ArgOp << " => "
+ << (IsAliasing ? "" : "no-") << "alias \n");
+
+ return IsAliasing;
+ }
+
+ bool isKnownNoAliasDueToNoAliasPreservation(
+ Attributor &A, AAResults *&AAR, const AAMemoryBehavior &MemBehaviorAA) {
+ // We can deduce "noalias" if the following conditions hold.
+ // (i) Associated value is assumed to be noalias in the definition.
+ // (ii) Associated value is assumed to be no-capture in all the uses
+ // possibly executed before this callsite.
+ // (iii) There is no other pointer argument which could alias with the
+ // value.
+
+ auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
+ const auto *DerefAA = A.getAAFor<AADereferenceable>(
+ *this, IRPosition::value(*O), DepClassTy::OPTIONAL);
+ return DerefAA ? DerefAA->getAssumedDereferenceableBytes() : 0;
+ };
+
+ const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
+ const Function *ScopeFn = VIRP.getAnchorScope();
+ // Check whether the value is captured in the scope using AANoCapture.
+ // Look at CFG and check only uses possibly executed before this
+ // callsite.
+ auto UsePred = [&](const Use &U, bool &Follow) -> bool {
+ Instruction *UserI = cast<Instruction>(U.getUser());
+
+ // If UserI is the curr instruction and there is a single potential use of
+ // the value in UserI we allow the use.
+ // TODO: We should inspect the operands and allow those that cannot alias
+ // with the value.
+ if (UserI == getCtxI() && UserI->getNumOperands() == 1)
+ return true;
+
+ if (ScopeFn) {
+ if (auto *CB = dyn_cast<CallBase>(UserI)) {
+ if (CB->isArgOperand(&U)) {
+
+ unsigned ArgNo = CB->getArgOperandNo(&U);
+
+ bool IsKnownNoCapture;
+ if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, IRPosition::callsite_argument(*CB, ArgNo),
+ DepClassTy::OPTIONAL, IsKnownNoCapture))
+ return true;
+ }
+ }
+
+ if (!AA::isPotentiallyReachable(
+ A, *UserI, *getCtxI(), *this, /* ExclusionSet */ nullptr,
+ [ScopeFn](const Function &Fn) { return &Fn != ScopeFn; }))
+ return true;
+ }
+
+ // TODO: We should track the capturing uses in AANoCapture but the problem
+ // is CGSCC runs. For those we would need to "allow" AANoCapture for
+ // a value in the module slice.
+ switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) {
+ case UseCaptureKind::NO_CAPTURE:
+ return true;
+ case UseCaptureKind::MAY_CAPTURE:
+ LLVM_DEBUG(dbgs() << "[AANoAliasCSArg] Unknown user: " << *UserI
+ << "\n");
+ return false;
+ case UseCaptureKind::PASSTHROUGH:
+ Follow = true;
+ return true;
+ }
+ llvm_unreachable("unknown UseCaptureKind");
+ };
+
+ bool IsKnownNoCapture;
+ const AANoCapture *NoCaptureAA = nullptr;
+ bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, VIRP, DepClassTy::NONE, IsKnownNoCapture, false, &NoCaptureAA);
+ if (!IsAssumedNoCapture &&
+ (!NoCaptureAA || !NoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
+ if (!A.checkForAllUses(UsePred, *this, getAssociatedValue())) {
+ LLVM_DEBUG(
+ dbgs() << "[AANoAliasCSArg] " << getAssociatedValue()
+ << " cannot be noalias as it is potentially captured\n");
+ return false;
+ }
+ }
+ if (NoCaptureAA)
+ A.recordDependence(*NoCaptureAA, *this, DepClassTy::OPTIONAL);
+
+ // Check there is no other pointer argument which could alias with the
+ // value passed at this call site.
+ // TODO: AbstractCallSite
+ const auto &CB = cast<CallBase>(getAnchorValue());
+ for (unsigned OtherArgNo = 0; OtherArgNo < CB.arg_size(); OtherArgNo++)
+ if (mayAliasWithArgument(A, AAR, MemBehaviorAA, CB, OtherArgNo))
+ return false;
+
+ return true;
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // If the argument is readnone we are done as there are no accesses via the
+ // argument.
+ auto *MemBehaviorAA =
+ A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
+ if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
+ A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);
+ return ChangeStatus::UNCHANGED;
+ }
+
+ bool IsKnownNoAlias;
+ const IRPosition &VIRP = IRPosition::value(getAssociatedValue());
+ if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
+ A, this, VIRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
+ LLVM_DEBUG(dbgs() << "[AANoAlias] " << getAssociatedValue()
+ << " is not no-alias at the definition\n");
+ return indicatePessimisticFixpoint();
+ }
+
+ AAResults *AAR = nullptr;
+ if (MemBehaviorAA &&
+ isKnownNoAliasDueToNoAliasPreservation(A, AAR, *MemBehaviorAA)) {
+ LLVM_DEBUG(
+ dbgs() << "[AANoAlias] No-Alias deduced via no-alias preservation\n");
+ return ChangeStatus::UNCHANGED;
+ }
+
+ return indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noalias) }
+};
+
+/// NoAlias attribute for function return value.
+struct AANoAliasReturned final : AANoAliasImpl {
+ AANoAliasReturned(const IRPosition &IRP, Attributor &A)
+ : AANoAliasImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+
+ auto CheckReturnValue = [&](Value &RV) -> bool {
+ if (Constant *C = dyn_cast<Constant>(&RV))
+ if (C->isNullValue() || isa<UndefValue>(C))
+ return true;
+
+ /// For now, we can only deduce noalias if we have call sites.
+ /// FIXME: add more support.
+ if (!isa<CallBase>(&RV))
+ return false;
+
+ const IRPosition &RVPos = IRPosition::value(RV);
+ bool IsKnownNoAlias;
+ if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
+ A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoAlias))
+ return false;
+
+ bool IsKnownNoCapture;
+ const AANoCapture *NoCaptureAA = nullptr;
+ bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, RVPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
+ &NoCaptureAA);
+ return IsAssumedNoCapture ||
+ (NoCaptureAA && NoCaptureAA->isAssumedNoCaptureMaybeReturned());
+ };
+
+ if (!A.checkForAllReturnedValues(CheckReturnValue, *this))
+ return indicatePessimisticFixpoint();
+
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noalias) }
+};
+
+/// NoAlias attribute deduction for a call site return value.
+struct AANoAliasCallSiteReturned final
+ : AACalleeToCallSite<AANoAlias, AANoAliasImpl> {
+ AANoAliasCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AANoAlias, AANoAliasImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noalias); }
+};
+} // namespace
+
+/// -------------------AAIsDead Function Attribute-----------------------
+
+namespace {
+struct AAIsDeadValueImpl : public AAIsDead {
+ AAIsDeadValueImpl(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
+
+ /// See AAIsDead::isAssumedDead().
+ bool isAssumedDead() const override { return isAssumed(IS_DEAD); }
+
+ /// See AAIsDead::isKnownDead().
+ bool isKnownDead() const override { return isKnown(IS_DEAD); }
+
+ /// See AAIsDead::isAssumedDead(BasicBlock *).
+ bool isAssumedDead(const BasicBlock *BB) const override { return false; }
+
+ /// See AAIsDead::isKnownDead(BasicBlock *).
+ bool isKnownDead(const BasicBlock *BB) const override { return false; }
+
+ /// See AAIsDead::isAssumedDead(Instruction *I).
+ bool isAssumedDead(const Instruction *I) const override {
+ return I == getCtxI() && isAssumedDead();
+ }
+
+ /// See AAIsDead::isKnownDead(Instruction *I).
+ bool isKnownDead(const Instruction *I) const override {
+ return isAssumedDead(I) && isKnownDead();
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return isAssumedDead() ? "assumed-dead" : "assumed-live";
+ }
+
+ /// Check if all uses are assumed dead.
+ bool areAllUsesAssumedDead(Attributor &A, Value &V) {
+ // Callers might not check the type, void has no uses.
+ if (V.getType()->isVoidTy() || V.use_empty())
+ return true;
+
+ // If we replace a value with a constant there are no uses left afterwards.
+ if (!isa<Constant>(V)) {
+ if (auto *I = dyn_cast<Instruction>(&V))
+ if (!A.isRunOn(*I->getFunction()))
+ return false;
+ bool UsedAssumedInformation = false;
+ std::optional<Constant *> C =
+ A.getAssumedConstant(V, *this, UsedAssumedInformation);
+ if (!C || *C)
+ return true;
+ }
+
+ auto UsePred = [&](const Use &U, bool &Follow) { return false; };
+ // Explicitly set the dependence class to required because we want a long
+ // chain of N dependent instructions to be considered live as soon as one is
+ // without going through N update cycles. This is not required for
+ // correctness.
+ return A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ false,
+ DepClassTy::REQUIRED,
+ /* IgnoreDroppableUses */ false);
+ }
+
+ /// Determine if \p I is assumed to be side-effect free.
+ bool isAssumedSideEffectFree(Attributor &A, Instruction *I) {
+ if (!I || wouldInstructionBeTriviallyDead(I))
+ return true;
+
+ auto *CB = dyn_cast<CallBase>(I);
+ if (!CB || isa<IntrinsicInst>(CB))
+ return false;
+
+ const IRPosition &CallIRP = IRPosition::callsite_function(*CB);
+
+ bool IsKnownNoUnwind;
+ if (!AA::hasAssumedIRAttr<Attribute::NoUnwind>(
+ A, this, CallIRP, DepClassTy::OPTIONAL, IsKnownNoUnwind))
+ return false;
+
+ bool IsKnown;
+ return AA::isAssumedReadOnly(A, CallIRP, *this, IsKnown);
+ }
+};
+
+struct AAIsDeadFloating : public AAIsDeadValueImpl {
+ AAIsDeadFloating(const IRPosition &IRP, Attributor &A)
+ : AAIsDeadValueImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ AAIsDeadValueImpl::initialize(A);
+
+ if (isa<UndefValue>(getAssociatedValue())) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+
+ Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
+ if (!isAssumedSideEffectFree(A, I)) {
+ if (!isa_and_nonnull<StoreInst>(I) && !isa_and_nonnull<FenceInst>(I))
+ indicatePessimisticFixpoint();
+ else
+ removeAssumedBits(HAS_NO_EFFECT);
+ }
+ }
+
+ bool isDeadFence(Attributor &A, FenceInst &FI) {
+ const auto *ExecDomainAA = A.lookupAAFor<AAExecutionDomain>(
+ IRPosition::function(*FI.getFunction()), *this, DepClassTy::NONE);
+ if (!ExecDomainAA || !ExecDomainAA->isNoOpFence(FI))
+ return false;
+ A.recordDependence(*ExecDomainAA, *this, DepClassTy::OPTIONAL);
+ return true;
+ }
+
+ bool isDeadStore(Attributor &A, StoreInst &SI,
+ SmallSetVector<Instruction *, 8> *AssumeOnlyInst = nullptr) {
+ // Lang ref now states volatile store is not UB/dead, let's skip them.
+ if (SI.isVolatile())
+ return false;
+
+ // If we are collecting assumes to be deleted we are in the manifest stage.
+ // It's problematic to collect the potential copies again now so we use the
+ // cached ones.
+ bool UsedAssumedInformation = false;
+ if (!AssumeOnlyInst) {
+ PotentialCopies.clear();
+ if (!AA::getPotentialCopiesOfStoredValue(A, SI, PotentialCopies, *this,
+ UsedAssumedInformation)) {
+ LLVM_DEBUG(
+ dbgs()
+ << "[AAIsDead] Could not determine potential copies of store!\n");
+ return false;
+ }
+ }
+ LLVM_DEBUG(dbgs() << "[AAIsDead] Store has " << PotentialCopies.size()
+ << " potential copies.\n");
+
+ InformationCache &InfoCache = A.getInfoCache();
+ return llvm::all_of(PotentialCopies, [&](Value *V) {
+ if (A.isAssumedDead(IRPosition::value(*V), this, nullptr,
+ UsedAssumedInformation))
+ return true;
+ if (auto *LI = dyn_cast<LoadInst>(V)) {
+ if (llvm::all_of(LI->uses(), [&](const Use &U) {
+ auto &UserI = cast<Instruction>(*U.getUser());
+ if (InfoCache.isOnlyUsedByAssume(UserI)) {
+ if (AssumeOnlyInst)
+ AssumeOnlyInst->insert(&UserI);
+ return true;
+ }
+ return A.isAssumedDead(U, this, nullptr, UsedAssumedInformation);
+ })) {
+ return true;
+ }
+ }
+ LLVM_DEBUG(dbgs() << "[AAIsDead] Potential copy " << *V
+ << " is assumed live!\n");
+ return false;
+ });
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
+ if (isa_and_nonnull<StoreInst>(I))
+ if (isValidState())
+ return "assumed-dead-store";
+ if (isa_and_nonnull<FenceInst>(I))
+ if (isValidState())
+ return "assumed-dead-fence";
+ return AAIsDeadValueImpl::getAsStr(A);
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
+ if (auto *SI = dyn_cast_or_null<StoreInst>(I)) {
+ if (!isDeadStore(A, *SI))
+ return indicatePessimisticFixpoint();
+ } else if (auto *FI = dyn_cast_or_null<FenceInst>(I)) {
+ if (!isDeadFence(A, *FI))
+ return indicatePessimisticFixpoint();
+ } else {
+ if (!isAssumedSideEffectFree(A, I))
+ return indicatePessimisticFixpoint();
+ if (!areAllUsesAssumedDead(A, getAssociatedValue()))
+ return indicatePessimisticFixpoint();
+ }
+ return ChangeStatus::UNCHANGED;
+ }
+
+ bool isRemovableStore() const override {
+ return isAssumed(IS_REMOVABLE) && isa<StoreInst>(&getAssociatedValue());
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ Value &V = getAssociatedValue();
+ if (auto *I = dyn_cast<Instruction>(&V)) {
+ // If we get here we basically know the users are all dead. We check if
+ // isAssumedSideEffectFree returns true here again because it might not be
+ // the case and only the users are dead but the instruction (=call) is
+ // still needed.
+ if (auto *SI = dyn_cast<StoreInst>(I)) {
+ SmallSetVector<Instruction *, 8> AssumeOnlyInst;
+ bool IsDead = isDeadStore(A, *SI, &AssumeOnlyInst);
+ (void)IsDead;
+ assert(IsDead && "Store was assumed to be dead!");
+ A.deleteAfterManifest(*I);
+ for (size_t i = 0; i < AssumeOnlyInst.size(); ++i) {
+ Instruction *AOI = AssumeOnlyInst[i];
+ for (auto *Usr : AOI->users())
+ AssumeOnlyInst.insert(cast<Instruction>(Usr));
+ A.deleteAfterManifest(*AOI);
+ }
+ return ChangeStatus::CHANGED;
+ }
+ if (auto *FI = dyn_cast<FenceInst>(I)) {
+ assert(isDeadFence(A, *FI));
+ A.deleteAfterManifest(*FI);
+ return ChangeStatus::CHANGED;
+ }
+ if (isAssumedSideEffectFree(A, I) && !isa<InvokeInst>(I)) {
+ A.deleteAfterManifest(*I);
+ return ChangeStatus::CHANGED;
+ }
+ }
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(IsDead)
+ }
+
+private:
+ // The potential copies of a dead store, used for deletion during manifest.
+ SmallSetVector<Value *, 4> PotentialCopies;
+};
+
+struct AAIsDeadArgument : public AAIsDeadFloating {
+ AAIsDeadArgument(const IRPosition &IRP, Attributor &A)
+ : AAIsDeadFloating(IRP, A) {}
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ Argument &Arg = *getAssociatedArgument();
+ if (A.isValidFunctionSignatureRewrite(Arg, /* ReplacementTypes */ {}))
+ if (A.registerFunctionSignatureRewrite(
+ Arg, /* ReplacementTypes */ {},
+ Attributor::ArgumentReplacementInfo::CalleeRepairCBTy{},
+ Attributor::ArgumentReplacementInfo::ACSRepairCBTy{})) {
+ return ChangeStatus::CHANGED;
+ }
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(IsDead) }
+};
+
+struct AAIsDeadCallSiteArgument : public AAIsDeadValueImpl {
+ AAIsDeadCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAIsDeadValueImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ AAIsDeadValueImpl::initialize(A);
+ if (isa<UndefValue>(getAssociatedValue()))
+ indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // TODO: Once we have call site specific value information we can provide
+ // call site specific liveness information and then it makes
+ // sense to specialize attributes for call sites arguments instead of
+ // redirecting requests to the callee argument.
+ Argument *Arg = getAssociatedArgument();
+ if (!Arg)
+ return indicatePessimisticFixpoint();
+ const IRPosition &ArgPos = IRPosition::argument(*Arg);
+ auto *ArgAA = A.getAAFor<AAIsDead>(*this, ArgPos, DepClassTy::REQUIRED);
+ if (!ArgAA)
+ return indicatePessimisticFixpoint();
+ return clampStateAndIndicateChange(getState(), ArgAA->getState());
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ CallBase &CB = cast<CallBase>(getAnchorValue());
+ Use &U = CB.getArgOperandUse(getCallSiteArgNo());
+ assert(!isa<UndefValue>(U.get()) &&
+ "Expected undef values to be filtered out!");
+ UndefValue &UV = *UndefValue::get(U->getType());
+ if (A.changeUseAfterManifest(U, UV))
+ return ChangeStatus::CHANGED;
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(IsDead) }
+};
+
+struct AAIsDeadCallSiteReturned : public AAIsDeadFloating {
+ AAIsDeadCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AAIsDeadFloating(IRP, A) {}
+
+ /// See AAIsDead::isAssumedDead().
+ bool isAssumedDead() const override {
+ return AAIsDeadFloating::isAssumedDead() && IsAssumedSideEffectFree;
+ }
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ AAIsDeadFloating::initialize(A);
+ if (isa<UndefValue>(getAssociatedValue())) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+
+ // We track this separately as a secondary state.
+ IsAssumedSideEffectFree = isAssumedSideEffectFree(A, getCtxI());
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ if (IsAssumedSideEffectFree && !isAssumedSideEffectFree(A, getCtxI())) {
+ IsAssumedSideEffectFree = false;
+ Changed = ChangeStatus::CHANGED;
+ }
+ if (!areAllUsesAssumedDead(A, getAssociatedValue()))
+ return indicatePessimisticFixpoint();
+ return Changed;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ if (IsAssumedSideEffectFree)
+ STATS_DECLTRACK_CSRET_ATTR(IsDead)
+ else
+ STATS_DECLTRACK_CSRET_ATTR(UnusedResult)
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return isAssumedDead()
+ ? "assumed-dead"
+ : (getAssumed() ? "assumed-dead-users" : "assumed-live");
+ }
+
+private:
+ bool IsAssumedSideEffectFree = true;
+};
+
+struct AAIsDeadReturned : public AAIsDeadValueImpl {
+ AAIsDeadReturned(const IRPosition &IRP, Attributor &A)
+ : AAIsDeadValueImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+
+ bool UsedAssumedInformation = false;
+ A.checkForAllInstructions([](Instruction &) { return true; }, *this,
+ {Instruction::Ret}, UsedAssumedInformation);
+
+ auto PredForCallSite = [&](AbstractCallSite ACS) {
+ if (ACS.isCallbackCall() || !ACS.getInstruction())
+ return false;
+ return areAllUsesAssumedDead(A, *ACS.getInstruction());
+ };
+
+ if (!A.checkForAllCallSites(PredForCallSite, *this, true,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ // TODO: Rewrite the signature to return void?
+ bool AnyChange = false;
+ UndefValue &UV = *UndefValue::get(getAssociatedFunction()->getReturnType());
+ auto RetInstPred = [&](Instruction &I) {
+ ReturnInst &RI = cast<ReturnInst>(I);
+ if (!isa<UndefValue>(RI.getReturnValue()))
+ AnyChange |= A.changeUseAfterManifest(RI.getOperandUse(0), UV);
+ return true;
+ };
+ bool UsedAssumedInformation = false;
+ A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
+ UsedAssumedInformation);
+ return AnyChange ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(IsDead) }
+};
+
+struct AAIsDeadFunction : public AAIsDead {
+ AAIsDeadFunction(const IRPosition &IRP, Attributor &A) : AAIsDead(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ Function *F = getAnchorScope();
+ assert(F && "Did expect an anchor function");
+ if (!isAssumedDeadInternalFunction(A)) {
+ ToBeExploredFrom.insert(&F->getEntryBlock().front());
+ assumeLive(A, F->getEntryBlock());
+ }
+ }
+
+ bool isAssumedDeadInternalFunction(Attributor &A) {
+ if (!getAnchorScope()->hasLocalLinkage())
+ return false;
+ bool UsedAssumedInformation = false;
+ return A.checkForAllCallSites([](AbstractCallSite) { return false; }, *this,
+ true, UsedAssumedInformation);
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return "Live[#BB " + std::to_string(AssumedLiveBlocks.size()) + "/" +
+ std::to_string(getAnchorScope()->size()) + "][#TBEP " +
+ std::to_string(ToBeExploredFrom.size()) + "][#KDE " +
+ std::to_string(KnownDeadEnds.size()) + "]";
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ assert(getState().isValidState() &&
+ "Attempted to manifest an invalid state!");
+
+ ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
+ Function &F = *getAnchorScope();
+
+ if (AssumedLiveBlocks.empty()) {
+ A.deleteAfterManifest(F);
+ return ChangeStatus::CHANGED;
+ }
+
+ // Flag to determine if we can change an invoke to a call assuming the
+ // callee is nounwind. This is not possible if the personality of the
+ // function allows to catch asynchronous exceptions.
+ bool Invoke2CallAllowed = !mayCatchAsynchronousExceptions(F);
+
+ KnownDeadEnds.set_union(ToBeExploredFrom);
+ for (const Instruction *DeadEndI : KnownDeadEnds) {
+ auto *CB = dyn_cast<CallBase>(DeadEndI);
+ if (!CB)
+ continue;
+ bool IsKnownNoReturn;
+ bool MayReturn = !AA::hasAssumedIRAttr<Attribute::NoReturn>(
+ A, this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL,
+ IsKnownNoReturn);
+ if (MayReturn && (!Invoke2CallAllowed || !isa<InvokeInst>(CB)))
+ continue;
+
+ if (auto *II = dyn_cast<InvokeInst>(DeadEndI))
+ A.registerInvokeWithDeadSuccessor(const_cast<InvokeInst &>(*II));
+ else
+ A.changeToUnreachableAfterManifest(
+ const_cast<Instruction *>(DeadEndI->getNextNode()));
+ HasChanged = ChangeStatus::CHANGED;
+ }
+
+ STATS_DECL(AAIsDead, BasicBlock, "Number of dead basic blocks deleted.");
+ for (BasicBlock &BB : F)
+ if (!AssumedLiveBlocks.count(&BB)) {
+ A.deleteAfterManifest(BB);
+ ++BUILD_STAT_NAME(AAIsDead, BasicBlock);
+ HasChanged = ChangeStatus::CHANGED;
+ }
+
+ return HasChanged;
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override;
+
+ bool isEdgeDead(const BasicBlock *From, const BasicBlock *To) const override {
+ assert(From->getParent() == getAnchorScope() &&
+ To->getParent() == getAnchorScope() &&
+ "Used AAIsDead of the wrong function");
+ return isValidState() && !AssumedLiveEdges.count(std::make_pair(From, To));
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {}
+
+ /// Returns true if the function is assumed dead.
+ bool isAssumedDead() const override { return false; }
+
+ /// See AAIsDead::isKnownDead().
+ bool isKnownDead() const override { return false; }
+
+ /// See AAIsDead::isAssumedDead(BasicBlock *).
+ bool isAssumedDead(const BasicBlock *BB) const override {
+ assert(BB->getParent() == getAnchorScope() &&
+ "BB must be in the same anchor scope function.");
+
+ if (!getAssumed())
+ return false;
+ return !AssumedLiveBlocks.count(BB);
+ }
+
+ /// See AAIsDead::isKnownDead(BasicBlock *).
+ bool isKnownDead(const BasicBlock *BB) const override {
+ return getKnown() && isAssumedDead(BB);
+ }
+
+ /// See AAIsDead::isAssumed(Instruction *I).
+ bool isAssumedDead(const Instruction *I) const override {
+ assert(I->getParent()->getParent() == getAnchorScope() &&
+ "Instruction must be in the same anchor scope function.");
+
+ if (!getAssumed())
+ return false;
+
+ // If it is not in AssumedLiveBlocks then it for sure dead.
+ // Otherwise, it can still be after noreturn call in a live block.
+ if (!AssumedLiveBlocks.count(I->getParent()))
+ return true;
+
+ // If it is not after a liveness barrier it is live.
+ const Instruction *PrevI = I->getPrevNode();
+ while (PrevI) {
+ if (KnownDeadEnds.count(PrevI) || ToBeExploredFrom.count(PrevI))
+ return true;
+ PrevI = PrevI->getPrevNode();
+ }
+ return false;
+ }
+
+ /// See AAIsDead::isKnownDead(Instruction *I).
+ bool isKnownDead(const Instruction *I) const override {
+ return getKnown() && isAssumedDead(I);
+ }
+
+ /// Assume \p BB is (partially) live now and indicate to the Attributor \p A
+ /// that internal function called from \p BB should now be looked at.
+ bool assumeLive(Attributor &A, const BasicBlock &BB) {
+ if (!AssumedLiveBlocks.insert(&BB).second)
+ return false;
+
+ // We assume that all of BB is (probably) live now and if there are calls to
+ // internal functions we will assume that those are now live as well. This
+ // is a performance optimization for blocks with calls to a lot of internal
+ // functions. It can however cause dead functions to be treated as live.
+ for (const Instruction &I : BB)
+ if (const auto *CB = dyn_cast<CallBase>(&I))
+ if (auto *F = dyn_cast_if_present<Function>(CB->getCalledOperand()))
+ if (F->hasLocalLinkage())
+ A.markLiveInternalFunction(*F);
+ return true;
+ }
+
+ /// Collection of instructions that need to be explored again, e.g., we
+ /// did assume they do not transfer control to (one of their) successors.
+ SmallSetVector<const Instruction *, 8> ToBeExploredFrom;
+
+ /// Collection of instructions that are known to not transfer control.
+ SmallSetVector<const Instruction *, 8> KnownDeadEnds;
+
+ /// Collection of all assumed live edges
+ DenseSet<std::pair<const BasicBlock *, const BasicBlock *>> AssumedLiveEdges;
+
+ /// Collection of all assumed live BasicBlocks.
+ DenseSet<const BasicBlock *> AssumedLiveBlocks;
+};
+
+static bool
+identifyAliveSuccessors(Attributor &A, const CallBase &CB,
+ AbstractAttribute &AA,
+ SmallVectorImpl<const Instruction *> &AliveSuccessors) {
+ const IRPosition &IPos = IRPosition::callsite_function(CB);
+
+ bool IsKnownNoReturn;
+ if (AA::hasAssumedIRAttr<Attribute::NoReturn>(
+ A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoReturn))
+ return !IsKnownNoReturn;
+ if (CB.isTerminator())
+ AliveSuccessors.push_back(&CB.getSuccessor(0)->front());
+ else
+ AliveSuccessors.push_back(CB.getNextNode());
+ return false;
+}
+
+static bool
+identifyAliveSuccessors(Attributor &A, const InvokeInst &II,
+ AbstractAttribute &AA,
+ SmallVectorImpl<const Instruction *> &AliveSuccessors) {
+ bool UsedAssumedInformation =
+ identifyAliveSuccessors(A, cast<CallBase>(II), AA, AliveSuccessors);
+
+ // First, determine if we can change an invoke to a call assuming the
+ // callee is nounwind. This is not possible if the personality of the
+ // function allows to catch asynchronous exceptions.
+ if (AAIsDeadFunction::mayCatchAsynchronousExceptions(*II.getFunction())) {
+ AliveSuccessors.push_back(&II.getUnwindDest()->front());
+ } else {
+ const IRPosition &IPos = IRPosition::callsite_function(II);
+
+ bool IsKnownNoUnwind;
+ if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
+ A, &AA, IPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
+ UsedAssumedInformation |= !IsKnownNoUnwind;
+ } else {
+ AliveSuccessors.push_back(&II.getUnwindDest()->front());
+ }
+ }
+ return UsedAssumedInformation;
+}
+
+static bool
+identifyAliveSuccessors(Attributor &A, const BranchInst &BI,
+ AbstractAttribute &AA,
+ SmallVectorImpl<const Instruction *> &AliveSuccessors) {
+ bool UsedAssumedInformation = false;
+ if (BI.getNumSuccessors() == 1) {
+ AliveSuccessors.push_back(&BI.getSuccessor(0)->front());
+ } else {
+ std::optional<Constant *> C =
+ A.getAssumedConstant(*BI.getCondition(), AA, UsedAssumedInformation);
+ if (!C || isa_and_nonnull<UndefValue>(*C)) {
+ // No value yet, assume both edges are dead.
+ } else if (isa_and_nonnull<ConstantInt>(*C)) {
+ const BasicBlock *SuccBB =
+ BI.getSuccessor(1 - cast<ConstantInt>(*C)->getValue().getZExtValue());
+ AliveSuccessors.push_back(&SuccBB->front());
+ } else {
+ AliveSuccessors.push_back(&BI.getSuccessor(0)->front());
+ AliveSuccessors.push_back(&BI.getSuccessor(1)->front());
+ UsedAssumedInformation = false;
+ }
+ }
+ return UsedAssumedInformation;
+}
+
+static bool
+identifyAliveSuccessors(Attributor &A, const SwitchInst &SI,
+ AbstractAttribute &AA,
+ SmallVectorImpl<const Instruction *> &AliveSuccessors) {
+ bool UsedAssumedInformation = false;
+ SmallVector<AA::ValueAndContext> Values;
+ if (!A.getAssumedSimplifiedValues(IRPosition::value(*SI.getCondition()), &AA,
+ Values, AA::AnyScope,
+ UsedAssumedInformation)) {
+ // Something went wrong, assume all successors are live.
+ for (const BasicBlock *SuccBB : successors(SI.getParent()))
+ AliveSuccessors.push_back(&SuccBB->front());
+ return false;
+ }
+
+ if (Values.empty() ||
+ (Values.size() == 1 &&
+ isa_and_nonnull<UndefValue>(Values.front().getValue()))) {
+ // No valid value yet, assume all edges are dead.
+ return UsedAssumedInformation;
+ }
+
+ Type &Ty = *SI.getCondition()->getType();
+ SmallPtrSet<ConstantInt *, 8> Constants;
+ auto CheckForConstantInt = [&](Value *V) {
+ if (auto *CI = dyn_cast_if_present<ConstantInt>(AA::getWithType(*V, Ty))) {
+ Constants.insert(CI);
+ return true;
+ }
+ return false;
+ };
+
+ if (!all_of(Values, [&](AA::ValueAndContext &VAC) {
+ return CheckForConstantInt(VAC.getValue());
+ })) {
+ for (const BasicBlock *SuccBB : successors(SI.getParent()))
+ AliveSuccessors.push_back(&SuccBB->front());
+ return UsedAssumedInformation;
+ }
+
+ unsigned MatchedCases = 0;
+ for (const auto &CaseIt : SI.cases()) {
+ if (Constants.count(CaseIt.getCaseValue())) {
+ ++MatchedCases;
+ AliveSuccessors.push_back(&CaseIt.getCaseSuccessor()->front());
+ }
+ }
+
+ // If all potential values have been matched, we will not visit the default
+ // case.
+ if (MatchedCases < Constants.size())
+ AliveSuccessors.push_back(&SI.getDefaultDest()->front());
+ return UsedAssumedInformation;
+}
+
+ChangeStatus AAIsDeadFunction::updateImpl(Attributor &A) {
+ ChangeStatus Change = ChangeStatus::UNCHANGED;
+
+ if (AssumedLiveBlocks.empty()) {
+ if (isAssumedDeadInternalFunction(A))
+ return ChangeStatus::UNCHANGED;
+
+ Function *F = getAnchorScope();
+ ToBeExploredFrom.insert(&F->getEntryBlock().front());
+ assumeLive(A, F->getEntryBlock());
+ Change = ChangeStatus::CHANGED;
+ }
+
+ LLVM_DEBUG(dbgs() << "[AAIsDead] Live [" << AssumedLiveBlocks.size() << "/"
+ << getAnchorScope()->size() << "] BBs and "
+ << ToBeExploredFrom.size() << " exploration points and "
+ << KnownDeadEnds.size() << " known dead ends\n");
+
+ // Copy and clear the list of instructions we need to explore from. It is
+ // refilled with instructions the next update has to look at.
+ SmallVector<const Instruction *, 8> Worklist(ToBeExploredFrom.begin(),
+ ToBeExploredFrom.end());
+ decltype(ToBeExploredFrom) NewToBeExploredFrom;
+
+ SmallVector<const Instruction *, 8> AliveSuccessors;
+ while (!Worklist.empty()) {
+ const Instruction *I = Worklist.pop_back_val();
+ LLVM_DEBUG(dbgs() << "[AAIsDead] Exploration inst: " << *I << "\n");
+
+ // Fast forward for uninteresting instructions. We could look for UB here
+ // though.
+ while (!I->isTerminator() && !isa<CallBase>(I))
+ I = I->getNextNode();
+
+ AliveSuccessors.clear();
+
+ bool UsedAssumedInformation = false;
+ switch (I->getOpcode()) {
+ // TODO: look for (assumed) UB to backwards propagate "deadness".
+ default:
+ assert(I->isTerminator() &&
+ "Expected non-terminators to be handled already!");
+ for (const BasicBlock *SuccBB : successors(I->getParent()))
+ AliveSuccessors.push_back(&SuccBB->front());
+ break;
+ case Instruction::Call:
+ UsedAssumedInformation = identifyAliveSuccessors(A, cast<CallInst>(*I),
+ *this, AliveSuccessors);
+ break;
+ case Instruction::Invoke:
+ UsedAssumedInformation = identifyAliveSuccessors(A, cast<InvokeInst>(*I),
+ *this, AliveSuccessors);
+ break;
+ case Instruction::Br:
+ UsedAssumedInformation = identifyAliveSuccessors(A, cast<BranchInst>(*I),
+ *this, AliveSuccessors);
+ break;
+ case Instruction::Switch:
+ UsedAssumedInformation = identifyAliveSuccessors(A, cast<SwitchInst>(*I),
+ *this, AliveSuccessors);
+ break;
+ }
+
+ if (UsedAssumedInformation) {
+ NewToBeExploredFrom.insert(I);
+ } else if (AliveSuccessors.empty() ||
+ (I->isTerminator() &&
+ AliveSuccessors.size() < I->getNumSuccessors())) {
+ if (KnownDeadEnds.insert(I))
+ Change = ChangeStatus::CHANGED;
+ }
+
+ LLVM_DEBUG(dbgs() << "[AAIsDead] #AliveSuccessors: "
+ << AliveSuccessors.size() << " UsedAssumedInformation: "
+ << UsedAssumedInformation << "\n");
+
+ for (const Instruction *AliveSuccessor : AliveSuccessors) {
+ if (!I->isTerminator()) {
+ assert(AliveSuccessors.size() == 1 &&
+ "Non-terminator expected to have a single successor!");
+ Worklist.push_back(AliveSuccessor);
+ } else {
+ // record the assumed live edge
+ auto Edge = std::make_pair(I->getParent(), AliveSuccessor->getParent());
+ if (AssumedLiveEdges.insert(Edge).second)
+ Change = ChangeStatus::CHANGED;
+ if (assumeLive(A, *AliveSuccessor->getParent()))
+ Worklist.push_back(AliveSuccessor);
+ }
+ }
+ }
+
+ // Check if the content of ToBeExploredFrom changed, ignore the order.
+ if (NewToBeExploredFrom.size() != ToBeExploredFrom.size() ||
+ llvm::any_of(NewToBeExploredFrom, [&](const Instruction *I) {
+ return !ToBeExploredFrom.count(I);
+ })) {
+ Change = ChangeStatus::CHANGED;
+ ToBeExploredFrom = std::move(NewToBeExploredFrom);
+ }
+
+ // If we know everything is live there is no need to query for liveness.
+ // Instead, indicating a pessimistic fixpoint will cause the state to be
+ // "invalid" and all queries to be answered conservatively without lookups.
+ // To be in this state we have to (1) finished the exploration and (3) not
+ // discovered any non-trivial dead end and (2) not ruled unreachable code
+ // dead.
+ if (ToBeExploredFrom.empty() &&
+ getAnchorScope()->size() == AssumedLiveBlocks.size() &&
+ llvm::all_of(KnownDeadEnds, [](const Instruction *DeadEndI) {
+ return DeadEndI->isTerminator() && DeadEndI->getNumSuccessors() == 0;
+ }))
+ return indicatePessimisticFixpoint();
+ return Change;
+}
+
+/// Liveness information for a call sites.
+struct AAIsDeadCallSite final : AAIsDeadFunction {
+ AAIsDeadCallSite(const IRPosition &IRP, Attributor &A)
+ : AAIsDeadFunction(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ // TODO: Once we have call site specific value information we can provide
+ // call site specific liveness information and then it makes
+ // sense to specialize attributes for call sites instead of
+ // redirecting requests to the callee.
+ llvm_unreachable("Abstract attributes for liveness are not "
+ "supported for call sites yet!");
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ return indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {}
+};
+} // namespace
+
+/// -------------------- Dereferenceable Argument Attribute --------------------
+
+namespace {
+struct AADereferenceableImpl : AADereferenceable {
+ AADereferenceableImpl(const IRPosition &IRP, Attributor &A)
+ : AADereferenceable(IRP, A) {}
+ using StateType = DerefState;
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ Value &V = *getAssociatedValue().stripPointerCasts();
+ SmallVector<Attribute, 4> Attrs;
+ A.getAttrs(getIRPosition(),
+ {Attribute::Dereferenceable, Attribute::DereferenceableOrNull},
+ Attrs, /* IgnoreSubsumingPositions */ false);
+ for (const Attribute &Attr : Attrs)
+ takeKnownDerefBytesMaximum(Attr.getValueAsInt());
+
+ // Ensure we initialize the non-null AA (if necessary).
+ bool IsKnownNonNull;
+ AA::hasAssumedIRAttr<Attribute::NonNull>(
+ A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNonNull);
+
+ bool CanBeNull, CanBeFreed;
+ takeKnownDerefBytesMaximum(V.getPointerDereferenceableBytes(
+ A.getDataLayout(), CanBeNull, CanBeFreed));
+
+ if (Instruction *CtxI = getCtxI())
+ followUsesInMBEC(*this, A, getState(), *CtxI);
+ }
+
+ /// See AbstractAttribute::getState()
+ /// {
+ StateType &getState() override { return *this; }
+ const StateType &getState() const override { return *this; }
+ /// }
+
+ /// Helper function for collecting accessed bytes in must-be-executed-context
+ void addAccessedBytesForUse(Attributor &A, const Use *U, const Instruction *I,
+ DerefState &State) {
+ const Value *UseV = U->get();
+ if (!UseV->getType()->isPointerTy())
+ return;
+
+ std::optional<MemoryLocation> Loc = MemoryLocation::getOrNone(I);
+ if (!Loc || Loc->Ptr != UseV || !Loc->Size.isPrecise() || I->isVolatile())
+ return;
+
+ int64_t Offset;
+ const Value *Base = GetPointerBaseWithConstantOffset(
+ Loc->Ptr, Offset, A.getDataLayout(), /*AllowNonInbounds*/ true);
+ if (Base && Base == &getAssociatedValue())
+ State.addAccessedBytes(Offset, Loc->Size.getValue());
+ }
+
+ /// See followUsesInMBEC
+ bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
+ AADereferenceable::StateType &State) {
+ bool IsNonNull = false;
+ bool TrackUse = false;
+ int64_t DerefBytes = getKnownNonNullAndDerefBytesForUse(
+ A, *this, getAssociatedValue(), U, I, IsNonNull, TrackUse);
+ LLVM_DEBUG(dbgs() << "[AADereferenceable] Deref bytes: " << DerefBytes
+ << " for instruction " << *I << "\n");
+
+ addAccessedBytesForUse(A, U, I, State);
+ State.takeKnownDerefBytesMaximum(DerefBytes);
+ return TrackUse;
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ ChangeStatus Change = AADereferenceable::manifest(A);
+ bool IsKnownNonNull;
+ bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
+ A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
+ if (IsAssumedNonNull &&
+ A.hasAttr(getIRPosition(), Attribute::DereferenceableOrNull)) {
+ A.removeAttrs(getIRPosition(), {Attribute::DereferenceableOrNull});
+ return ChangeStatus::CHANGED;
+ }
+ return Change;
+ }
+
+ void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
+ SmallVectorImpl<Attribute> &Attrs) const override {
+ // TODO: Add *_globally support
+ bool IsKnownNonNull;
+ bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
+ A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
+ if (IsAssumedNonNull)
+ Attrs.emplace_back(Attribute::getWithDereferenceableBytes(
+ Ctx, getAssumedDereferenceableBytes()));
+ else
+ Attrs.emplace_back(Attribute::getWithDereferenceableOrNullBytes(
+ Ctx, getAssumedDereferenceableBytes()));
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ if (!getAssumedDereferenceableBytes())
+ return "unknown-dereferenceable";
+ bool IsKnownNonNull;
+ bool IsAssumedNonNull = false;
+ if (A)
+ IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
+ *A, this, getIRPosition(), DepClassTy::NONE, IsKnownNonNull);
+ return std::string("dereferenceable") +
+ (IsAssumedNonNull ? "" : "_or_null") +
+ (isAssumedGlobal() ? "_globally" : "") + "<" +
+ std::to_string(getKnownDereferenceableBytes()) + "-" +
+ std::to_string(getAssumedDereferenceableBytes()) + ">" +
+ (!A ? " [non-null is unknown]" : "");
+ }
+};
+
+/// Dereferenceable attribute for a floating value.
+struct AADereferenceableFloating : AADereferenceableImpl {
+ AADereferenceableFloating(const IRPosition &IRP, Attributor &A)
+ : AADereferenceableImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ bool Stripped;
+ bool UsedAssumedInformation = false;
+ SmallVector<AA::ValueAndContext> Values;
+ if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
+ AA::AnyScope, UsedAssumedInformation)) {
+ Values.push_back({getAssociatedValue(), getCtxI()});
+ Stripped = false;
+ } else {
+ Stripped = Values.size() != 1 ||
+ Values.front().getValue() != &getAssociatedValue();
+ }
+
+ const DataLayout &DL = A.getDataLayout();
+ DerefState T;
+
+ auto VisitValueCB = [&](const Value &V) -> bool {
+ unsigned IdxWidth =
+ DL.getIndexSizeInBits(V.getType()->getPointerAddressSpace());
+ APInt Offset(IdxWidth, 0);
+ const Value *Base = stripAndAccumulateOffsets(
+ A, *this, &V, DL, Offset, /* GetMinOffset */ false,
+ /* AllowNonInbounds */ true);
+
+ const auto *AA = A.getAAFor<AADereferenceable>(
+ *this, IRPosition::value(*Base), DepClassTy::REQUIRED);
+ int64_t DerefBytes = 0;
+ if (!AA || (!Stripped && this == AA)) {
+ // Use IR information if we did not strip anything.
+ // TODO: track globally.
+ bool CanBeNull, CanBeFreed;
+ DerefBytes =
+ Base->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
+ T.GlobalState.indicatePessimisticFixpoint();
+ } else {
+ const DerefState &DS = AA->getState();
+ DerefBytes = DS.DerefBytesState.getAssumed();
+ T.GlobalState &= DS.GlobalState;
+ }
+
+ // For now we do not try to "increase" dereferenceability due to negative
+ // indices as we first have to come up with code to deal with loops and
+ // for overflows of the dereferenceable bytes.
+ int64_t OffsetSExt = Offset.getSExtValue();
+ if (OffsetSExt < 0)
+ OffsetSExt = 0;
+
+ T.takeAssumedDerefBytesMinimum(
+ std::max(int64_t(0), DerefBytes - OffsetSExt));
+
+ if (this == AA) {
+ if (!Stripped) {
+ // If nothing was stripped IR information is all we got.
+ T.takeKnownDerefBytesMaximum(
+ std::max(int64_t(0), DerefBytes - OffsetSExt));
+ T.indicatePessimisticFixpoint();
+ } else if (OffsetSExt > 0) {
+ // If something was stripped but there is circular reasoning we look
+ // for the offset. If it is positive we basically decrease the
+ // dereferenceable bytes in a circular loop now, which will simply
+ // drive them down to the known value in a very slow way which we
+ // can accelerate.
+ T.indicatePessimisticFixpoint();
+ }
+ }
+
+ return T.isValidState();
+ };
+
+ for (const auto &VAC : Values)
+ if (!VisitValueCB(*VAC.getValue()))
+ return indicatePessimisticFixpoint();
+
+ return clampStateAndIndicateChange(getState(), T);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(dereferenceable)
+ }
+};
+
+/// Dereferenceable attribute for a return value.
+struct AADereferenceableReturned final
+ : AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl> {
+ using Base =
+ AAReturnedFromReturnedValues<AADereferenceable, AADereferenceableImpl>;
+ AADereferenceableReturned(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FNRET_ATTR(dereferenceable)
+ }
+};
+
+/// Dereferenceable attribute for an argument
+struct AADereferenceableArgument final
+ : AAArgumentFromCallSiteArguments<AADereferenceable,
+ AADereferenceableImpl> {
+ using Base =
+ AAArgumentFromCallSiteArguments<AADereferenceable, AADereferenceableImpl>;
+ AADereferenceableArgument(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_ARG_ATTR(dereferenceable)
+ }
+};
+
+/// Dereferenceable attribute for a call site argument.
+struct AADereferenceableCallSiteArgument final : AADereferenceableFloating {
+ AADereferenceableCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AADereferenceableFloating(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSARG_ATTR(dereferenceable)
+ }
+};
+
+/// Dereferenceable attribute deduction for a call site return value.
+struct AADereferenceableCallSiteReturned final
+ : AACalleeToCallSite<AADereferenceable, AADereferenceableImpl> {
+ using Base = AACalleeToCallSite<AADereferenceable, AADereferenceableImpl>;
+ AADereferenceableCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CS_ATTR(dereferenceable);
+ }
+};
+} // namespace
+
+// ------------------------ Align Argument Attribute ------------------------
+
+namespace {
+static unsigned getKnownAlignForUse(Attributor &A, AAAlign &QueryingAA,
+ Value &AssociatedValue, const Use *U,
+ const Instruction *I, bool &TrackUse) {
+ // We need to follow common pointer manipulation uses to the accesses they
+ // feed into.
+ if (isa<CastInst>(I)) {
+ // Follow all but ptr2int casts.
+ TrackUse = !isa<PtrToIntInst>(I);
+ return 0;
+ }
+ if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
+ if (GEP->hasAllConstantIndices())
+ TrackUse = true;
+ return 0;
+ }
+
+ MaybeAlign MA;
+ if (const auto *CB = dyn_cast<CallBase>(I)) {
+ if (CB->isBundleOperand(U) || CB->isCallee(U))
+ return 0;
+
+ unsigned ArgNo = CB->getArgOperandNo(U);
+ IRPosition IRP = IRPosition::callsite_argument(*CB, ArgNo);
+ // As long as we only use known information there is no need to track
+ // dependences here.
+ auto *AlignAA = A.getAAFor<AAAlign>(QueryingAA, IRP, DepClassTy::NONE);
+ if (AlignAA)
+ MA = MaybeAlign(AlignAA->getKnownAlign());
+ }
+
+ const DataLayout &DL = A.getDataLayout();
+ const Value *UseV = U->get();
+ if (auto *SI = dyn_cast<StoreInst>(I)) {
+ if (SI->getPointerOperand() == UseV)
+ MA = SI->getAlign();
+ } else if (auto *LI = dyn_cast<LoadInst>(I)) {
+ if (LI->getPointerOperand() == UseV)
+ MA = LI->getAlign();
+ } else if (auto *AI = dyn_cast<AtomicRMWInst>(I)) {
+ if (AI->getPointerOperand() == UseV)
+ MA = AI->getAlign();
+ } else if (auto *AI = dyn_cast<AtomicCmpXchgInst>(I)) {
+ if (AI->getPointerOperand() == UseV)
+ MA = AI->getAlign();
+ }
+
+ if (!MA || *MA <= QueryingAA.getKnownAlign())
+ return 0;
+
+ unsigned Alignment = MA->value();
+ int64_t Offset;
+
+ if (const Value *Base = GetPointerBaseWithConstantOffset(UseV, Offset, DL)) {
+ if (Base == &AssociatedValue) {
+ // BasePointerAddr + Offset = Alignment * Q for some integer Q.
+ // So we can say that the maximum power of two which is a divisor of
+ // gcd(Offset, Alignment) is an alignment.
+
+ uint32_t gcd = std::gcd(uint32_t(abs((int32_t)Offset)), Alignment);
+ Alignment = llvm::bit_floor(gcd);
+ }
+ }
+
+ return Alignment;
+}
+
+struct AAAlignImpl : AAAlign {
+ AAAlignImpl(const IRPosition &IRP, Attributor &A) : AAAlign(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ SmallVector<Attribute, 4> Attrs;
+ A.getAttrs(getIRPosition(), {Attribute::Alignment}, Attrs);
+ for (const Attribute &Attr : Attrs)
+ takeKnownMaximum(Attr.getValueAsInt());
+
+ Value &V = *getAssociatedValue().stripPointerCasts();
+ takeKnownMaximum(V.getPointerAlignment(A.getDataLayout()).value());
+
+ if (Instruction *CtxI = getCtxI())
+ followUsesInMBEC(*this, A, getState(), *CtxI);
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ ChangeStatus LoadStoreChanged = ChangeStatus::UNCHANGED;
+
+ // Check for users that allow alignment annotations.
+ Value &AssociatedValue = getAssociatedValue();
+ for (const Use &U : AssociatedValue.uses()) {
+ if (auto *SI = dyn_cast<StoreInst>(U.getUser())) {
+ if (SI->getPointerOperand() == &AssociatedValue)
+ if (SI->getAlign() < getAssumedAlign()) {
+ STATS_DECLTRACK(AAAlign, Store,
+ "Number of times alignment added to a store");
+ SI->setAlignment(getAssumedAlign());
+ LoadStoreChanged = ChangeStatus::CHANGED;
+ }
+ } else if (auto *LI = dyn_cast<LoadInst>(U.getUser())) {
+ if (LI->getPointerOperand() == &AssociatedValue)
+ if (LI->getAlign() < getAssumedAlign()) {
+ LI->setAlignment(getAssumedAlign());
+ STATS_DECLTRACK(AAAlign, Load,
+ "Number of times alignment added to a load");
+ LoadStoreChanged = ChangeStatus::CHANGED;
+ }
+ }
+ }
+
+ ChangeStatus Changed = AAAlign::manifest(A);
+
+ Align InheritAlign =
+ getAssociatedValue().getPointerAlignment(A.getDataLayout());
+ if (InheritAlign >= getAssumedAlign())
+ return LoadStoreChanged;
+ return Changed | LoadStoreChanged;
+ }
+
+ // TODO: Provide a helper to determine the implied ABI alignment and check in
+ // the existing manifest method and a new one for AAAlignImpl that value
+ // to avoid making the alignment explicit if it did not improve.
+
+ /// See AbstractAttribute::getDeducedAttributes
+ void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
+ SmallVectorImpl<Attribute> &Attrs) const override {
+ if (getAssumedAlign() > 1)
+ Attrs.emplace_back(
+ Attribute::getWithAlignment(Ctx, Align(getAssumedAlign())));
+ }
+
+ /// See followUsesInMBEC
+ bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
+ AAAlign::StateType &State) {
+ bool TrackUse = false;
+
+ unsigned int KnownAlign =
+ getKnownAlignForUse(A, *this, getAssociatedValue(), U, I, TrackUse);
+ State.takeKnownMaximum(KnownAlign);
+
+ return TrackUse;
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return "align<" + std::to_string(getKnownAlign().value()) + "-" +
+ std::to_string(getAssumedAlign().value()) + ">";
+ }
+};
+
+/// Align attribute for a floating value.
+struct AAAlignFloating : AAAlignImpl {
+ AAAlignFloating(const IRPosition &IRP, Attributor &A) : AAAlignImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ const DataLayout &DL = A.getDataLayout();
+
+ bool Stripped;
+ bool UsedAssumedInformation = false;
+ SmallVector<AA::ValueAndContext> Values;
+ if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
+ AA::AnyScope, UsedAssumedInformation)) {
+ Values.push_back({getAssociatedValue(), getCtxI()});
+ Stripped = false;
+ } else {
+ Stripped = Values.size() != 1 ||
+ Values.front().getValue() != &getAssociatedValue();
+ }
+
+ StateType T;
+ auto VisitValueCB = [&](Value &V) -> bool {
+ if (isa<UndefValue>(V) || isa<ConstantPointerNull>(V))
+ return true;
+ const auto *AA = A.getAAFor<AAAlign>(*this, IRPosition::value(V),
+ DepClassTy::REQUIRED);
+ if (!AA || (!Stripped && this == AA)) {
+ int64_t Offset;
+ unsigned Alignment = 1;
+ if (const Value *Base =
+ GetPointerBaseWithConstantOffset(&V, Offset, DL)) {
+ // TODO: Use AAAlign for the base too.
+ Align PA = Base->getPointerAlignment(DL);
+ // BasePointerAddr + Offset = Alignment * Q for some integer Q.
+ // So we can say that the maximum power of two which is a divisor of
+ // gcd(Offset, Alignment) is an alignment.
+
+ uint32_t gcd =
+ std::gcd(uint32_t(abs((int32_t)Offset)), uint32_t(PA.value()));
+ Alignment = llvm::bit_floor(gcd);
+ } else {
+ Alignment = V.getPointerAlignment(DL).value();
+ }
+ // Use only IR information if we did not strip anything.
+ T.takeKnownMaximum(Alignment);
+ T.indicatePessimisticFixpoint();
+ } else {
+ // Use abstract attribute information.
+ const AAAlign::StateType &DS = AA->getState();
+ T ^= DS;
+ }
+ return T.isValidState();
+ };
+
+ for (const auto &VAC : Values) {
+ if (!VisitValueCB(*VAC.getValue()))
+ return indicatePessimisticFixpoint();
+ }
+
+ // TODO: If we know we visited all incoming values, thus no are assumed
+ // dead, we can take the known information from the state T.
+ return clampStateAndIndicateChange(getState(), T);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FLOATING_ATTR(align) }
+};
+
+/// Align attribute for function return value.
+struct AAAlignReturned final
+ : AAReturnedFromReturnedValues<AAAlign, AAAlignImpl> {
+ using Base = AAReturnedFromReturnedValues<AAAlign, AAAlignImpl>;
+ AAAlignReturned(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(aligned) }
+};
+
+/// Align attribute for function argument.
+struct AAAlignArgument final
+ : AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl> {
+ using Base = AAArgumentFromCallSiteArguments<AAAlign, AAAlignImpl>;
+ AAAlignArgument(const IRPosition &IRP, Attributor &A) : Base(IRP, A) {}
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ // If the associated argument is involved in a must-tail call we give up
+ // because we would need to keep the argument alignments of caller and
+ // callee in-sync. Just does not seem worth the trouble right now.
+ if (A.getInfoCache().isInvolvedInMustTailCall(*getAssociatedArgument()))
+ return ChangeStatus::UNCHANGED;
+ return Base::manifest(A);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(aligned) }
+};
+
+struct AAAlignCallSiteArgument final : AAAlignFloating {
+ AAAlignCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAAlignFloating(IRP, A) {}
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ // If the associated argument is involved in a must-tail call we give up
+ // because we would need to keep the argument alignments of caller and
+ // callee in-sync. Just does not seem worth the trouble right now.
+ if (Argument *Arg = getAssociatedArgument())
+ if (A.getInfoCache().isInvolvedInMustTailCall(*Arg))
+ return ChangeStatus::UNCHANGED;
+ ChangeStatus Changed = AAAlignImpl::manifest(A);
+ Align InheritAlign =
+ getAssociatedValue().getPointerAlignment(A.getDataLayout());
+ if (InheritAlign >= getAssumedAlign())
+ Changed = ChangeStatus::UNCHANGED;
+ return Changed;
+ }
+
+ /// See AbstractAttribute::updateImpl(Attributor &A).
+ ChangeStatus updateImpl(Attributor &A) override {
+ ChangeStatus Changed = AAAlignFloating::updateImpl(A);
+ if (Argument *Arg = getAssociatedArgument()) {
+ // We only take known information from the argument
+ // so we do not need to track a dependence.
+ const auto *ArgAlignAA = A.getAAFor<AAAlign>(
+ *this, IRPosition::argument(*Arg), DepClassTy::NONE);
+ if (ArgAlignAA)
+ takeKnownMaximum(ArgAlignAA->getKnownAlign().value());
+ }
+ return Changed;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(aligned) }
+};
+
+/// Align attribute deduction for a call site return value.
+struct AAAlignCallSiteReturned final
+ : AACalleeToCallSite<AAAlign, AAAlignImpl> {
+ using Base = AACalleeToCallSite<AAAlign, AAAlignImpl>;
+ AAAlignCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(align); }
+};
+} // namespace
+
+/// ------------------ Function No-Return Attribute ----------------------------
+namespace {
+struct AANoReturnImpl : public AANoReturn {
+ AANoReturnImpl(const IRPosition &IRP, Attributor &A) : AANoReturn(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ bool IsKnown;
+ assert(!AA::hasAssumedIRAttr<Attribute::NoReturn>(
+ A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
+ (void)IsKnown;
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "noreturn" : "may-return";
+ }
+
+ /// See AbstractAttribute::updateImpl(Attributor &A).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto CheckForNoReturn = [](Instruction &) { return false; };
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllInstructions(CheckForNoReturn, *this,
+ {(unsigned)Instruction::Ret},
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+ return ChangeStatus::UNCHANGED;
+ }
+};
+
+struct AANoReturnFunction final : AANoReturnImpl {
+ AANoReturnFunction(const IRPosition &IRP, Attributor &A)
+ : AANoReturnImpl(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(noreturn) }
+};
+
+/// NoReturn attribute deduction for a call sites.
+struct AANoReturnCallSite final
+ : AACalleeToCallSite<AANoReturn, AANoReturnImpl> {
+ AANoReturnCallSite(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AANoReturn, AANoReturnImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(noreturn); }
+};
+} // namespace
+
+/// ----------------------- Instance Info ---------------------------------
+
+namespace {
+/// A class to hold the state of for no-capture attributes.
+struct AAInstanceInfoImpl : public AAInstanceInfo {
+ AAInstanceInfoImpl(const IRPosition &IRP, Attributor &A)
+ : AAInstanceInfo(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ Value &V = getAssociatedValue();
+ if (auto *C = dyn_cast<Constant>(&V)) {
+ if (C->isThreadDependent())
+ indicatePessimisticFixpoint();
+ else
+ indicateOptimisticFixpoint();
+ return;
+ }
+ if (auto *CB = dyn_cast<CallBase>(&V))
+ if (CB->arg_size() == 0 && !CB->mayHaveSideEffects() &&
+ !CB->mayReadFromMemory()) {
+ indicateOptimisticFixpoint();
+ return;
+ }
+ if (auto *I = dyn_cast<Instruction>(&V)) {
+ const auto *CI =
+ A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
+ *I->getFunction());
+ if (mayBeInCycle(CI, I, /* HeaderOnly */ false)) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+ }
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+
+ Value &V = getAssociatedValue();
+ const Function *Scope = nullptr;
+ if (auto *I = dyn_cast<Instruction>(&V))
+ Scope = I->getFunction();
+ if (auto *A = dyn_cast<Argument>(&V)) {
+ Scope = A->getParent();
+ if (!Scope->hasLocalLinkage())
+ return Changed;
+ }
+ if (!Scope)
+ return indicateOptimisticFixpoint();
+
+ bool IsKnownNoRecurse;
+ if (AA::hasAssumedIRAttr<Attribute::NoRecurse>(
+ A, this, IRPosition::function(*Scope), DepClassTy::OPTIONAL,
+ IsKnownNoRecurse))
+ return Changed;
+
+ auto UsePred = [&](const Use &U, bool &Follow) {
+ const Instruction *UserI = dyn_cast<Instruction>(U.getUser());
+ if (!UserI || isa<GetElementPtrInst>(UserI) || isa<CastInst>(UserI) ||
+ isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
+ Follow = true;
+ return true;
+ }
+ if (isa<LoadInst>(UserI) || isa<CmpInst>(UserI) ||
+ (isa<StoreInst>(UserI) &&
+ cast<StoreInst>(UserI)->getValueOperand() != U.get()))
+ return true;
+ if (auto *CB = dyn_cast<CallBase>(UserI)) {
+ // This check is not guaranteeing uniqueness but for now that we cannot
+ // end up with two versions of \p U thinking it was one.
+ auto *Callee = dyn_cast_if_present<Function>(CB->getCalledOperand());
+ if (!Callee || !Callee->hasLocalLinkage())
+ return true;
+ if (!CB->isArgOperand(&U))
+ return false;
+ const auto *ArgInstanceInfoAA = A.getAAFor<AAInstanceInfo>(
+ *this, IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U)),
+ DepClassTy::OPTIONAL);
+ if (!ArgInstanceInfoAA ||
+ !ArgInstanceInfoAA->isAssumedUniqueForAnalysis())
+ return false;
+ // If this call base might reach the scope again we might forward the
+ // argument back here. This is very conservative.
+ if (AA::isPotentiallyReachable(
+ A, *CB, *Scope, *this, /* ExclusionSet */ nullptr,
+ [Scope](const Function &Fn) { return &Fn != Scope; }))
+ return false;
+ return true;
+ }
+ return false;
+ };
+
+ auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
+ if (auto *SI = dyn_cast<StoreInst>(OldU.getUser())) {
+ auto *Ptr = SI->getPointerOperand()->stripPointerCasts();
+ if ((isa<AllocaInst>(Ptr) || isNoAliasCall(Ptr)) &&
+ AA::isDynamicallyUnique(A, *this, *Ptr))
+ return true;
+ }
+ return false;
+ };
+
+ if (!A.checkForAllUses(UsePred, *this, V, /* CheckBBLivenessOnly */ true,
+ DepClassTy::OPTIONAL,
+ /* IgnoreDroppableUses */ true, EquivalentUseCB))
+ return indicatePessimisticFixpoint();
+
+ return Changed;
+ }
+
+ /// See AbstractState::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return isAssumedUniqueForAnalysis() ? "<unique [fAa]>" : "<unknown>";
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {}
+};
+
+/// InstanceInfo attribute for floating values.
+struct AAInstanceInfoFloating : AAInstanceInfoImpl {
+ AAInstanceInfoFloating(const IRPosition &IRP, Attributor &A)
+ : AAInstanceInfoImpl(IRP, A) {}
+};
+
+/// NoCapture attribute for function arguments.
+struct AAInstanceInfoArgument final : AAInstanceInfoFloating {
+ AAInstanceInfoArgument(const IRPosition &IRP, Attributor &A)
+ : AAInstanceInfoFloating(IRP, A) {}
+};
+
+/// InstanceInfo attribute for call site arguments.
+struct AAInstanceInfoCallSiteArgument final : AAInstanceInfoImpl {
+ AAInstanceInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAInstanceInfoImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // TODO: Once we have call site specific value information we can provide
+ // call site specific liveness information and then it makes
+ // sense to specialize attributes for call sites arguments instead of
+ // redirecting requests to the callee argument.
+ Argument *Arg = getAssociatedArgument();
+ if (!Arg)
+ return indicatePessimisticFixpoint();
+ const IRPosition &ArgPos = IRPosition::argument(*Arg);
+ auto *ArgAA =
+ A.getAAFor<AAInstanceInfo>(*this, ArgPos, DepClassTy::REQUIRED);
+ if (!ArgAA)
+ return indicatePessimisticFixpoint();
+ return clampStateAndIndicateChange(getState(), ArgAA->getState());
+ }
+};
+
+/// InstanceInfo attribute for function return value.
+struct AAInstanceInfoReturned final : AAInstanceInfoImpl {
+ AAInstanceInfoReturned(const IRPosition &IRP, Attributor &A)
+ : AAInstanceInfoImpl(IRP, A) {
+ llvm_unreachable("InstanceInfo is not applicable to function returns!");
+ }
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ llvm_unreachable("InstanceInfo is not applicable to function returns!");
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ llvm_unreachable("InstanceInfo is not applicable to function returns!");
+ }
+};
+
+/// InstanceInfo attribute deduction for a call site return value.
+struct AAInstanceInfoCallSiteReturned final : AAInstanceInfoFloating {
+ AAInstanceInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AAInstanceInfoFloating(IRP, A) {}
+};
+} // namespace
+
+/// ----------------------- Variable Capturing ---------------------------------
+bool AANoCapture::isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions) {
+ assert(ImpliedAttributeKind == Attribute::NoCapture &&
+ "Unexpected attribute kind");
+ Value &V = IRP.getAssociatedValue();
+ if (!IRP.isArgumentPosition())
+ return V.use_empty();
+
+ // You cannot "capture" null in the default address space.
+ if (isa<UndefValue>(V) || (isa<ConstantPointerNull>(V) &&
+ V.getType()->getPointerAddressSpace() == 0)) {
+ return true;
+ }
+
+ if (A.hasAttr(IRP, {Attribute::NoCapture},
+ /* IgnoreSubsumingPositions */ true, Attribute::NoCapture))
+ return true;
+
+ if (IRP.getPositionKind() == IRP_CALL_SITE_ARGUMENT)
+ if (Argument *Arg = IRP.getAssociatedArgument())
+ if (A.hasAttr(IRPosition::argument(*Arg),
+ {Attribute::NoCapture, Attribute::ByVal},
+ /* IgnoreSubsumingPositions */ true)) {
+ A.manifestAttrs(IRP,
+ Attribute::get(V.getContext(), Attribute::NoCapture));
+ return true;
+ }
+
+ if (const Function *F = IRP.getAssociatedFunction()) {
+ // Check what state the associated function can actually capture.
+ AANoCapture::StateType State;
+ determineFunctionCaptureCapabilities(IRP, *F, State);
+ if (State.isKnown(NO_CAPTURE)) {
+ A.manifestAttrs(IRP,
+ Attribute::get(V.getContext(), Attribute::NoCapture));
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/// Set the NOT_CAPTURED_IN_MEM and NOT_CAPTURED_IN_RET bits in \p Known
+/// depending on the ability of the function associated with \p IRP to capture
+/// state in memory and through "returning/throwing", respectively.
+void AANoCapture::determineFunctionCaptureCapabilities(const IRPosition &IRP,
+ const Function &F,
+ BitIntegerState &State) {
+ // TODO: Once we have memory behavior attributes we should use them here.
+
+ // If we know we cannot communicate or write to memory, we do not care about
+ // ptr2int anymore.
+ bool ReadOnly = F.onlyReadsMemory();
+ bool NoThrow = F.doesNotThrow();
+ bool IsVoidReturn = F.getReturnType()->isVoidTy();
+ if (ReadOnly && NoThrow && IsVoidReturn) {
+ State.addKnownBits(NO_CAPTURE);
+ return;
+ }
+
+ // A function cannot capture state in memory if it only reads memory, it can
+ // however return/throw state and the state might be influenced by the
+ // pointer value, e.g., loading from a returned pointer might reveal a bit.
+ if (ReadOnly)
+ State.addKnownBits(NOT_CAPTURED_IN_MEM);
+
+ // A function cannot communicate state back if it does not through
+ // exceptions and doesn not return values.
+ if (NoThrow && IsVoidReturn)
+ State.addKnownBits(NOT_CAPTURED_IN_RET);
+
+ // Check existing "returned" attributes.
+ int ArgNo = IRP.getCalleeArgNo();
+ if (!NoThrow || ArgNo < 0 ||
+ !F.getAttributes().hasAttrSomewhere(Attribute::Returned))
+ return;
+
+ for (unsigned U = 0, E = F.arg_size(); U < E; ++U)
+ if (F.hasParamAttribute(U, Attribute::Returned)) {
+ if (U == unsigned(ArgNo))
+ State.removeAssumedBits(NOT_CAPTURED_IN_RET);
+ else if (ReadOnly)
+ State.addKnownBits(NO_CAPTURE);
+ else
+ State.addKnownBits(NOT_CAPTURED_IN_RET);
+ break;
+ }
+}
+
+namespace {
+/// A class to hold the state of for no-capture attributes.
+struct AANoCaptureImpl : public AANoCapture {
+ AANoCaptureImpl(const IRPosition &IRP, Attributor &A) : AANoCapture(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ bool IsKnown;
+ assert(!AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, nullptr, getIRPosition(), DepClassTy::NONE, IsKnown));
+ (void)IsKnown;
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override;
+
+ /// see AbstractAttribute::isAssumedNoCaptureMaybeReturned(...).
+ void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
+ SmallVectorImpl<Attribute> &Attrs) const override {
+ if (!isAssumedNoCaptureMaybeReturned())
+ return;
+
+ if (isArgumentPosition()) {
+ if (isAssumedNoCapture())
+ Attrs.emplace_back(Attribute::get(Ctx, Attribute::NoCapture));
+ else if (ManifestInternal)
+ Attrs.emplace_back(Attribute::get(Ctx, "no-capture-maybe-returned"));
+ }
+ }
+
+ /// See AbstractState::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ if (isKnownNoCapture())
+ return "known not-captured";
+ if (isAssumedNoCapture())
+ return "assumed not-captured";
+ if (isKnownNoCaptureMaybeReturned())
+ return "known not-captured-maybe-returned";
+ if (isAssumedNoCaptureMaybeReturned())
+ return "assumed not-captured-maybe-returned";
+ return "assumed-captured";
+ }
+
+ /// Check the use \p U and update \p State accordingly. Return true if we
+ /// should continue to update the state.
+ bool checkUse(Attributor &A, AANoCapture::StateType &State, const Use &U,
+ bool &Follow) {
+ Instruction *UInst = cast<Instruction>(U.getUser());
+ LLVM_DEBUG(dbgs() << "[AANoCapture] Check use: " << *U.get() << " in "
+ << *UInst << "\n");
+
+ // Deal with ptr2int by following uses.
+ if (isa<PtrToIntInst>(UInst)) {
+ LLVM_DEBUG(dbgs() << " - ptr2int assume the worst!\n");
+ return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
+ /* Return */ true);
+ }
+
+ // For stores we already checked if we can follow them, if they make it
+ // here we give up.
+ if (isa<StoreInst>(UInst))
+ return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
+ /* Return */ true);
+
+ // Explicitly catch return instructions.
+ if (isa<ReturnInst>(UInst)) {
+ if (UInst->getFunction() == getAnchorScope())
+ return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
+ /* Return */ true);
+ return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
+ /* Return */ true);
+ }
+
+ // For now we only use special logic for call sites. However, the tracker
+ // itself knows about a lot of other non-capturing cases already.
+ auto *CB = dyn_cast<CallBase>(UInst);
+ if (!CB || !CB->isArgOperand(&U))
+ return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
+ /* Return */ true);
+
+ unsigned ArgNo = CB->getArgOperandNo(&U);
+ const IRPosition &CSArgPos = IRPosition::callsite_argument(*CB, ArgNo);
+ // If we have a abstract no-capture attribute for the argument we can use
+ // it to justify a non-capture attribute here. This allows recursion!
+ bool IsKnownNoCapture;
+ const AANoCapture *ArgNoCaptureAA = nullptr;
+ bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, CSArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
+ &ArgNoCaptureAA);
+ if (IsAssumedNoCapture)
+ return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
+ /* Return */ false);
+ if (ArgNoCaptureAA && ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned()) {
+ Follow = true;
+ return isCapturedIn(State, /* Memory */ false, /* Integer */ false,
+ /* Return */ false);
+ }
+
+ // Lastly, we could not find a reason no-capture can be assumed so we don't.
+ return isCapturedIn(State, /* Memory */ true, /* Integer */ true,
+ /* Return */ true);
+ }
+
+ /// Update \p State according to \p CapturedInMem, \p CapturedInInt, and
+ /// \p CapturedInRet, then return true if we should continue updating the
+ /// state.
+ static bool isCapturedIn(AANoCapture::StateType &State, bool CapturedInMem,
+ bool CapturedInInt, bool CapturedInRet) {
+ LLVM_DEBUG(dbgs() << " - captures [Mem " << CapturedInMem << "|Int "
+ << CapturedInInt << "|Ret " << CapturedInRet << "]\n");
+ if (CapturedInMem)
+ State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_MEM);
+ if (CapturedInInt)
+ State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_INT);
+ if (CapturedInRet)
+ State.removeAssumedBits(AANoCapture::NOT_CAPTURED_IN_RET);
+ return State.isAssumed(AANoCapture::NO_CAPTURE_MAYBE_RETURNED);
+ }
+};
+
+ChangeStatus AANoCaptureImpl::updateImpl(Attributor &A) {
+ const IRPosition &IRP = getIRPosition();
+ Value *V = isArgumentPosition() ? IRP.getAssociatedArgument()
+ : &IRP.getAssociatedValue();
+ if (!V)
+ return indicatePessimisticFixpoint();
+
+ const Function *F =
+ isArgumentPosition() ? IRP.getAssociatedFunction() : IRP.getAnchorScope();
+ assert(F && "Expected a function!");
+ const IRPosition &FnPos = IRPosition::function(*F);
+
+ AANoCapture::StateType T;
+
+ // Readonly means we cannot capture through memory.
+ bool IsKnown;
+ if (AA::isAssumedReadOnly(A, FnPos, *this, IsKnown)) {
+ T.addKnownBits(NOT_CAPTURED_IN_MEM);
+ if (IsKnown)
+ addKnownBits(NOT_CAPTURED_IN_MEM);
+ }
+
+ // Make sure all returned values are different than the underlying value.
+ // TODO: we could do this in a more sophisticated way inside
+ // AAReturnedValues, e.g., track all values that escape through returns
+ // directly somehow.
+ auto CheckReturnedArgs = [&](bool &UsedAssumedInformation) {
+ SmallVector<AA::ValueAndContext> Values;
+ if (!A.getAssumedSimplifiedValues(IRPosition::returned(*F), this, Values,
+ AA::ValueScope::Intraprocedural,
+ UsedAssumedInformation))
+ return false;
+ bool SeenConstant = false;
+ for (const AA::ValueAndContext &VAC : Values) {
+ if (isa<Constant>(VAC.getValue())) {
+ if (SeenConstant)
+ return false;
+ SeenConstant = true;
+ } else if (!isa<Argument>(VAC.getValue()) ||
+ VAC.getValue() == getAssociatedArgument())
+ return false;
+ }
+ return true;
+ };
+
+ bool IsKnownNoUnwind;
+ if (AA::hasAssumedIRAttr<Attribute::NoUnwind>(
+ A, this, FnPos, DepClassTy::OPTIONAL, IsKnownNoUnwind)) {
+ bool IsVoidTy = F->getReturnType()->isVoidTy();
+ bool UsedAssumedInformation = false;
+ if (IsVoidTy || CheckReturnedArgs(UsedAssumedInformation)) {
+ T.addKnownBits(NOT_CAPTURED_IN_RET);
+ if (T.isKnown(NOT_CAPTURED_IN_MEM))
+ return ChangeStatus::UNCHANGED;
+ if (IsKnownNoUnwind && (IsVoidTy || !UsedAssumedInformation)) {
+ addKnownBits(NOT_CAPTURED_IN_RET);
+ if (isKnown(NOT_CAPTURED_IN_MEM))
+ return indicateOptimisticFixpoint();
+ }
+ }
+ }
+
+ auto IsDereferenceableOrNull = [&](Value *O, const DataLayout &DL) {
+ const auto *DerefAA = A.getAAFor<AADereferenceable>(
+ *this, IRPosition::value(*O), DepClassTy::OPTIONAL);
+ return DerefAA && DerefAA->getAssumedDereferenceableBytes();
+ };
+
+ auto UseCheck = [&](const Use &U, bool &Follow) -> bool {
+ switch (DetermineUseCaptureKind(U, IsDereferenceableOrNull)) {
+ case UseCaptureKind::NO_CAPTURE:
+ return true;
+ case UseCaptureKind::MAY_CAPTURE:
+ return checkUse(A, T, U, Follow);
+ case UseCaptureKind::PASSTHROUGH:
+ Follow = true;
+ return true;
+ }
+ llvm_unreachable("Unexpected use capture kind!");
+ };
+
+ if (!A.checkForAllUses(UseCheck, *this, *V))
+ return indicatePessimisticFixpoint();
+
+ AANoCapture::StateType &S = getState();
+ auto Assumed = S.getAssumed();
+ S.intersectAssumedBits(T.getAssumed());
+ if (!isAssumedNoCaptureMaybeReturned())
+ return indicatePessimisticFixpoint();
+ return Assumed == S.getAssumed() ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+}
+
+/// NoCapture attribute for function arguments.
+struct AANoCaptureArgument final : AANoCaptureImpl {
+ AANoCaptureArgument(const IRPosition &IRP, Attributor &A)
+ : AANoCaptureImpl(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nocapture) }
+};
+
+/// NoCapture attribute for call site arguments.
+struct AANoCaptureCallSiteArgument final : AANoCaptureImpl {
+ AANoCaptureCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AANoCaptureImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // TODO: Once we have call site specific value information we can provide
+ // call site specific liveness information and then it makes
+ // sense to specialize attributes for call sites arguments instead of
+ // redirecting requests to the callee argument.
+ Argument *Arg = getAssociatedArgument();
+ if (!Arg)
+ return indicatePessimisticFixpoint();
+ const IRPosition &ArgPos = IRPosition::argument(*Arg);
+ bool IsKnownNoCapture;
+ const AANoCapture *ArgAA = nullptr;
+ if (AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, ArgPos, DepClassTy::REQUIRED, IsKnownNoCapture, false,
+ &ArgAA))
+ return ChangeStatus::UNCHANGED;
+ if (!ArgAA || !ArgAA->isAssumedNoCaptureMaybeReturned())
+ return indicatePessimisticFixpoint();
+ return clampStateAndIndicateChange(getState(), ArgAA->getState());
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override{STATS_DECLTRACK_CSARG_ATTR(nocapture)};
+};
+
+/// NoCapture attribute for floating values.
+struct AANoCaptureFloating final : AANoCaptureImpl {
+ AANoCaptureFloating(const IRPosition &IRP, Attributor &A)
+ : AANoCaptureImpl(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(nocapture)
+ }
+};
+
+/// NoCapture attribute for function return value.
+struct AANoCaptureReturned final : AANoCaptureImpl {
+ AANoCaptureReturned(const IRPosition &IRP, Attributor &A)
+ : AANoCaptureImpl(IRP, A) {
+ llvm_unreachable("NoCapture is not applicable to function returns!");
+ }
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ llvm_unreachable("NoCapture is not applicable to function returns!");
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ llvm_unreachable("NoCapture is not applicable to function returns!");
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {}
+};
+
+/// NoCapture attribute deduction for a call site return value.
+struct AANoCaptureCallSiteReturned final : AANoCaptureImpl {
+ AANoCaptureCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AANoCaptureImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ const Function *F = getAnchorScope();
+ // Check what state the associated function can actually capture.
+ determineFunctionCaptureCapabilities(getIRPosition(), *F, *this);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSRET_ATTR(nocapture)
+ }
+};
+} // namespace
+
+/// ------------------ Value Simplify Attribute ----------------------------
+
+bool ValueSimplifyStateType::unionAssumed(std::optional<Value *> Other) {
+ // FIXME: Add a typecast support.
+ SimplifiedAssociatedValue = AA::combineOptionalValuesInAAValueLatice(
+ SimplifiedAssociatedValue, Other, Ty);
+ if (SimplifiedAssociatedValue == std::optional<Value *>(nullptr))
+ return false;
+
+ LLVM_DEBUG({
+ if (SimplifiedAssociatedValue)
+ dbgs() << "[ValueSimplify] is assumed to be "
+ << **SimplifiedAssociatedValue << "\n";
+ else
+ dbgs() << "[ValueSimplify] is assumed to be <none>\n";
+ });
+ return true;
+}
+
+namespace {
+struct AAValueSimplifyImpl : AAValueSimplify {
+ AAValueSimplifyImpl(const IRPosition &IRP, Attributor &A)
+ : AAValueSimplify(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ if (getAssociatedValue().getType()->isVoidTy())
+ indicatePessimisticFixpoint();
+ if (A.hasSimplificationCallback(getIRPosition()))
+ indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ LLVM_DEBUG({
+ dbgs() << "SAV: " << (bool)SimplifiedAssociatedValue << " ";
+ if (SimplifiedAssociatedValue && *SimplifiedAssociatedValue)
+ dbgs() << "SAV: " << **SimplifiedAssociatedValue << " ";
+ });
+ return isValidState() ? (isAtFixpoint() ? "simplified" : "maybe-simple")
+ : "not-simple";
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {}
+
+ /// See AAValueSimplify::getAssumedSimplifiedValue()
+ std::optional<Value *>
+ getAssumedSimplifiedValue(Attributor &A) const override {
+ return SimplifiedAssociatedValue;
+ }
+
+ /// Ensure the return value is \p V with type \p Ty, if not possible return
+ /// nullptr. If \p Check is true we will only verify such an operation would
+ /// suceed and return a non-nullptr value if that is the case. No IR is
+ /// generated or modified.
+ static Value *ensureType(Attributor &A, Value &V, Type &Ty, Instruction *CtxI,
+ bool Check) {
+ if (auto *TypedV = AA::getWithType(V, Ty))
+ return TypedV;
+ if (CtxI && V.getType()->canLosslesslyBitCastTo(&Ty))
+ return Check ? &V
+ : BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
+ &V, &Ty, "", CtxI->getIterator());
+ return nullptr;
+ }
+
+ /// Reproduce \p I with type \p Ty or return nullptr if that is not posisble.
+ /// If \p Check is true we will only verify such an operation would suceed and
+ /// return a non-nullptr value if that is the case. No IR is generated or
+ /// modified.
+ static Value *reproduceInst(Attributor &A,
+ const AbstractAttribute &QueryingAA,
+ Instruction &I, Type &Ty, Instruction *CtxI,
+ bool Check, ValueToValueMapTy &VMap) {
+ assert(CtxI && "Cannot reproduce an instruction without context!");
+ if (Check && (I.mayReadFromMemory() ||
+ !isSafeToSpeculativelyExecute(&I, CtxI, /* DT */ nullptr,
+ /* TLI */ nullptr)))
+ return nullptr;
+ for (Value *Op : I.operands()) {
+ Value *NewOp = reproduceValue(A, QueryingAA, *Op, Ty, CtxI, Check, VMap);
+ if (!NewOp) {
+ assert(Check && "Manifest of new value unexpectedly failed!");
+ return nullptr;
+ }
+ if (!Check)
+ VMap[Op] = NewOp;
+ }
+ if (Check)
+ return &I;
+
+ Instruction *CloneI = I.clone();
+ // TODO: Try to salvage debug information here.
+ CloneI->setDebugLoc(DebugLoc());
+ VMap[&I] = CloneI;
+ CloneI->insertBefore(CtxI);
+ RemapInstruction(CloneI, VMap);
+ return CloneI;
+ }
+
+ /// Reproduce \p V with type \p Ty or return nullptr if that is not posisble.
+ /// If \p Check is true we will only verify such an operation would suceed and
+ /// return a non-nullptr value if that is the case. No IR is generated or
+ /// modified.
+ static Value *reproduceValue(Attributor &A,
+ const AbstractAttribute &QueryingAA, Value &V,
+ Type &Ty, Instruction *CtxI, bool Check,
+ ValueToValueMapTy &VMap) {
+ if (const auto &NewV = VMap.lookup(&V))
+ return NewV;
+ bool UsedAssumedInformation = false;
+ std::optional<Value *> SimpleV = A.getAssumedSimplified(
+ V, QueryingAA, UsedAssumedInformation, AA::Interprocedural);
+ if (!SimpleV.has_value())
+ return PoisonValue::get(&Ty);
+ Value *EffectiveV = &V;
+ if (*SimpleV)
+ EffectiveV = *SimpleV;
+ if (auto *C = dyn_cast<Constant>(EffectiveV))
+ return C;
+ if (CtxI && AA::isValidAtPosition(AA::ValueAndContext(*EffectiveV, *CtxI),
+ A.getInfoCache()))
+ return ensureType(A, *EffectiveV, Ty, CtxI, Check);
+ if (auto *I = dyn_cast<Instruction>(EffectiveV))
+ if (Value *NewV = reproduceInst(A, QueryingAA, *I, Ty, CtxI, Check, VMap))
+ return ensureType(A, *NewV, Ty, CtxI, Check);
+ return nullptr;
+ }
+
+ /// Return a value we can use as replacement for the associated one, or
+ /// nullptr if we don't have one that makes sense.
+ Value *manifestReplacementValue(Attributor &A, Instruction *CtxI) const {
+ Value *NewV = SimplifiedAssociatedValue
+ ? *SimplifiedAssociatedValue
+ : UndefValue::get(getAssociatedType());
+ if (NewV && NewV != &getAssociatedValue()) {
+ ValueToValueMapTy VMap;
+ // First verify we can reprduce the value with the required type at the
+ // context location before we actually start modifying the IR.
+ if (reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,
+ /* CheckOnly */ true, VMap))
+ return reproduceValue(A, *this, *NewV, *getAssociatedType(), CtxI,
+ /* CheckOnly */ false, VMap);
+ }
+ return nullptr;
+ }
+
+ /// Helper function for querying AAValueSimplify and updating candidate.
+ /// \param IRP The value position we are trying to unify with SimplifiedValue
+ bool checkAndUpdate(Attributor &A, const AbstractAttribute &QueryingAA,
+ const IRPosition &IRP, bool Simplify = true) {
+ bool UsedAssumedInformation = false;
+ std::optional<Value *> QueryingValueSimplified = &IRP.getAssociatedValue();
+ if (Simplify)
+ QueryingValueSimplified = A.getAssumedSimplified(
+ IRP, QueryingAA, UsedAssumedInformation, AA::Interprocedural);
+ return unionAssumed(QueryingValueSimplified);
+ }
+
+ /// Returns a candidate is found or not
+ template <typename AAType> bool askSimplifiedValueFor(Attributor &A) {
+ if (!getAssociatedValue().getType()->isIntegerTy())
+ return false;
+
+ // This will also pass the call base context.
+ const auto *AA =
+ A.getAAFor<AAType>(*this, getIRPosition(), DepClassTy::NONE);
+ if (!AA)
+ return false;
+
+ std::optional<Constant *> COpt = AA->getAssumedConstant(A);
+
+ if (!COpt) {
+ SimplifiedAssociatedValue = std::nullopt;
+ A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);
+ return true;
+ }
+ if (auto *C = *COpt) {
+ SimplifiedAssociatedValue = C;
+ A.recordDependence(*AA, *this, DepClassTy::OPTIONAL);
+ return true;
+ }
+ return false;
+ }
+
+ bool askSimplifiedValueForOtherAAs(Attributor &A) {
+ if (askSimplifiedValueFor<AAValueConstantRange>(A))
+ return true;
+ if (askSimplifiedValueFor<AAPotentialConstantValues>(A))
+ return true;
+ return false;
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ for (auto &U : getAssociatedValue().uses()) {
+ // Check if we need to adjust the insertion point to make sure the IR is
+ // valid.
+ Instruction *IP = dyn_cast<Instruction>(U.getUser());
+ if (auto *PHI = dyn_cast_or_null<PHINode>(IP))
+ IP = PHI->getIncomingBlock(U)->getTerminator();
+ if (auto *NewV = manifestReplacementValue(A, IP)) {
+ LLVM_DEBUG(dbgs() << "[ValueSimplify] " << getAssociatedValue()
+ << " -> " << *NewV << " :: " << *this << "\n");
+ if (A.changeUseAfterManifest(U, *NewV))
+ Changed = ChangeStatus::CHANGED;
+ }
+ }
+
+ return Changed | AAValueSimplify::manifest(A);
+ }
+
+ /// See AbstractState::indicatePessimisticFixpoint(...).
+ ChangeStatus indicatePessimisticFixpoint() override {
+ SimplifiedAssociatedValue = &getAssociatedValue();
+ return AAValueSimplify::indicatePessimisticFixpoint();
+ }
+};
+
+struct AAValueSimplifyArgument final : AAValueSimplifyImpl {
+ AAValueSimplifyArgument(const IRPosition &IRP, Attributor &A)
+ : AAValueSimplifyImpl(IRP, A) {}
+
+ void initialize(Attributor &A) override {
+ AAValueSimplifyImpl::initialize(A);
+ if (A.hasAttr(getIRPosition(),
+ {Attribute::InAlloca, Attribute::Preallocated,
+ Attribute::StructRet, Attribute::Nest, Attribute::ByVal},
+ /* IgnoreSubsumingPositions */ true))
+ indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // Byval is only replacable if it is readonly otherwise we would write into
+ // the replaced value and not the copy that byval creates implicitly.
+ Argument *Arg = getAssociatedArgument();
+ if (Arg->hasByValAttr()) {
+ // TODO: We probably need to verify synchronization is not an issue, e.g.,
+ // there is no race by not copying a constant byval.
+ bool IsKnown;
+ if (!AA::isAssumedReadOnly(A, getIRPosition(), *this, IsKnown))
+ return indicatePessimisticFixpoint();
+ }
+
+ auto Before = SimplifiedAssociatedValue;
+
+ auto PredForCallSite = [&](AbstractCallSite ACS) {
+ const IRPosition &ACSArgPos =
+ IRPosition::callsite_argument(ACS, getCallSiteArgNo());
+ // Check if a coresponding argument was found or if it is on not
+ // associated (which can happen for callback calls).
+ if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
+ return false;
+
+ // Simplify the argument operand explicitly and check if the result is
+ // valid in the current scope. This avoids refering to simplified values
+ // in other functions, e.g., we don't want to say a an argument in a
+ // static function is actually an argument in a different function.
+ bool UsedAssumedInformation = false;
+ std::optional<Constant *> SimpleArgOp =
+ A.getAssumedConstant(ACSArgPos, *this, UsedAssumedInformation);
+ if (!SimpleArgOp)
+ return true;
+ if (!*SimpleArgOp)
+ return false;
+ if (!AA::isDynamicallyUnique(A, *this, **SimpleArgOp))
+ return false;
+ return unionAssumed(*SimpleArgOp);
+ };
+
+ // Generate a answer specific to a call site context.
+ bool Success;
+ bool UsedAssumedInformation = false;
+ if (hasCallBaseContext() &&
+ getCallBaseContext()->getCalledOperand() == Arg->getParent())
+ Success = PredForCallSite(
+ AbstractCallSite(&getCallBaseContext()->getCalledOperandUse()));
+ else
+ Success = A.checkForAllCallSites(PredForCallSite, *this, true,
+ UsedAssumedInformation);
+
+ if (!Success)
+ if (!askSimplifiedValueForOtherAAs(A))
+ return indicatePessimisticFixpoint();
+
+ // If a candidate was found in this update, return CHANGED.
+ return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
+ : ChangeStatus ::CHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_ARG_ATTR(value_simplify)
+ }
+};
+
+struct AAValueSimplifyReturned : AAValueSimplifyImpl {
+ AAValueSimplifyReturned(const IRPosition &IRP, Attributor &A)
+ : AAValueSimplifyImpl(IRP, A) {}
+
+ /// See AAValueSimplify::getAssumedSimplifiedValue()
+ std::optional<Value *>
+ getAssumedSimplifiedValue(Attributor &A) const override {
+ if (!isValidState())
+ return nullptr;
+ return SimplifiedAssociatedValue;
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto Before = SimplifiedAssociatedValue;
+
+ auto ReturnInstCB = [&](Instruction &I) {
+ auto &RI = cast<ReturnInst>(I);
+ return checkAndUpdate(
+ A, *this,
+ IRPosition::value(*RI.getReturnValue(), getCallBaseContext()));
+ };
+
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllInstructions(ReturnInstCB, *this, {Instruction::Ret},
+ UsedAssumedInformation))
+ if (!askSimplifiedValueForOtherAAs(A))
+ return indicatePessimisticFixpoint();
+
+ // If a candidate was found in this update, return CHANGED.
+ return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
+ : ChangeStatus ::CHANGED;
+ }
+
+ ChangeStatus manifest(Attributor &A) override {
+ // We queried AAValueSimplify for the returned values so they will be
+ // replaced if a simplified form was found. Nothing to do here.
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FNRET_ATTR(value_simplify)
+ }
+};
+
+struct AAValueSimplifyFloating : AAValueSimplifyImpl {
+ AAValueSimplifyFloating(const IRPosition &IRP, Attributor &A)
+ : AAValueSimplifyImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ AAValueSimplifyImpl::initialize(A);
+ Value &V = getAnchorValue();
+
+ // TODO: add other stuffs
+ if (isa<Constant>(V))
+ indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto Before = SimplifiedAssociatedValue;
+ if (!askSimplifiedValueForOtherAAs(A))
+ return indicatePessimisticFixpoint();
+
+ // If a candidate was found in this update, return CHANGED.
+ return Before == SimplifiedAssociatedValue ? ChangeStatus::UNCHANGED
+ : ChangeStatus ::CHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(value_simplify)
+ }
+};
+
+struct AAValueSimplifyFunction : AAValueSimplifyImpl {
+ AAValueSimplifyFunction(const IRPosition &IRP, Attributor &A)
+ : AAValueSimplifyImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ SimplifiedAssociatedValue = nullptr;
+ indicateOptimisticFixpoint();
+ }
+ /// See AbstractAttribute::initialize(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ llvm_unreachable(
+ "AAValueSimplify(Function|CallSite)::updateImpl will not be called");
+ }
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FN_ATTR(value_simplify)
+ }
+};
+
+struct AAValueSimplifyCallSite : AAValueSimplifyFunction {
+ AAValueSimplifyCallSite(const IRPosition &IRP, Attributor &A)
+ : AAValueSimplifyFunction(IRP, A) {}
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CS_ATTR(value_simplify)
+ }
+};
+
+struct AAValueSimplifyCallSiteReturned : AAValueSimplifyImpl {
+ AAValueSimplifyCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AAValueSimplifyImpl(IRP, A) {}
+
+ void initialize(Attributor &A) override {
+ AAValueSimplifyImpl::initialize(A);
+ Function *Fn = getAssociatedFunction();
+ assert(Fn && "Did expect an associted function");
+ for (Argument &Arg : Fn->args()) {
+ if (Arg.hasReturnedAttr()) {
+ auto IRP = IRPosition::callsite_argument(*cast<CallBase>(getCtxI()),
+ Arg.getArgNo());
+ if (IRP.getPositionKind() == IRPosition::IRP_CALL_SITE_ARGUMENT &&
+ checkAndUpdate(A, *this, IRP))
+ indicateOptimisticFixpoint();
+ else
+ indicatePessimisticFixpoint();
+ return;
+ }
+ }
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ return indicatePessimisticFixpoint();
+ }
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSRET_ATTR(value_simplify)
+ }
+};
+
+struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
+ AAValueSimplifyCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAValueSimplifyFloating(IRP, A) {}
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ // TODO: We should avoid simplification duplication to begin with.
+ auto *FloatAA = A.lookupAAFor<AAValueSimplify>(
+ IRPosition::value(getAssociatedValue()), this, DepClassTy::NONE);
+ if (FloatAA && FloatAA->getState().isValidState())
+ return Changed;
+
+ if (auto *NewV = manifestReplacementValue(A, getCtxI())) {
+ Use &U = cast<CallBase>(&getAnchorValue())
+ ->getArgOperandUse(getCallSiteArgNo());
+ if (A.changeUseAfterManifest(U, *NewV))
+ Changed = ChangeStatus::CHANGED;
+ }
+
+ return Changed | AAValueSimplify::manifest(A);
+ }
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSARG_ATTR(value_simplify)
+ }
+};
+} // namespace
+
+/// ----------------------- Heap-To-Stack Conversion ---------------------------
+namespace {
+struct AAHeapToStackFunction final : public AAHeapToStack {
+
+ struct AllocationInfo {
+ /// The call that allocates the memory.
+ CallBase *const CB;
+
+ /// The library function id for the allocation.
+ LibFunc LibraryFunctionId = NotLibFunc;
+
+ /// The status wrt. a rewrite.
+ enum {
+ STACK_DUE_TO_USE,
+ STACK_DUE_TO_FREE,
+ INVALID,
+ } Status = STACK_DUE_TO_USE;
+
+ /// Flag to indicate if we encountered a use that might free this allocation
+ /// but which is not in the deallocation infos.
+ bool HasPotentiallyFreeingUnknownUses = false;
+
+ /// Flag to indicate that we should place the new alloca in the function
+ /// entry block rather than where the call site (CB) is.
+ bool MoveAllocaIntoEntry = true;
+
+ /// The set of free calls that use this allocation.
+ SmallSetVector<CallBase *, 1> PotentialFreeCalls{};
+ };
+
+ struct DeallocationInfo {
+ /// The call that deallocates the memory.
+ CallBase *const CB;
+ /// The value freed by the call.
+ Value *FreedOp;
+
+ /// Flag to indicate if we don't know all objects this deallocation might
+ /// free.
+ bool MightFreeUnknownObjects = false;
+
+ /// The set of allocation calls that are potentially freed.
+ SmallSetVector<CallBase *, 1> PotentialAllocationCalls{};
+ };
+
+ AAHeapToStackFunction(const IRPosition &IRP, Attributor &A)
+ : AAHeapToStack(IRP, A) {}
+
+ ~AAHeapToStackFunction() {
+ // Ensure we call the destructor so we release any memory allocated in the
+ // sets.
+ for (auto &It : AllocationInfos)
+ It.second->~AllocationInfo();
+ for (auto &It : DeallocationInfos)
+ It.second->~DeallocationInfo();
+ }
+
+ void initialize(Attributor &A) override {
+ AAHeapToStack::initialize(A);
+
+ const Function *F = getAnchorScope();
+ const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
+
+ auto AllocationIdentifierCB = [&](Instruction &I) {
+ CallBase *CB = dyn_cast<CallBase>(&I);
+ if (!CB)
+ return true;
+ if (Value *FreedOp = getFreedOperand(CB, TLI)) {
+ DeallocationInfos[CB] = new (A.Allocator) DeallocationInfo{CB, FreedOp};
+ return true;
+ }
+ // To do heap to stack, we need to know that the allocation itself is
+ // removable once uses are rewritten, and that we can initialize the
+ // alloca to the same pattern as the original allocation result.
+ if (isRemovableAlloc(CB, TLI)) {
+ auto *I8Ty = Type::getInt8Ty(CB->getParent()->getContext());
+ if (nullptr != getInitialValueOfAllocation(CB, TLI, I8Ty)) {
+ AllocationInfo *AI = new (A.Allocator) AllocationInfo{CB};
+ AllocationInfos[CB] = AI;
+ if (TLI)
+ TLI->getLibFunc(*CB, AI->LibraryFunctionId);
+ }
+ }
+ return true;
+ };
+
+ bool UsedAssumedInformation = false;
+ bool Success = A.checkForAllCallLikeInstructions(
+ AllocationIdentifierCB, *this, UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ false,
+ /* CheckPotentiallyDead */ true);
+ (void)Success;
+ assert(Success && "Did not expect the call base visit callback to fail!");
+
+ Attributor::SimplifictionCallbackTy SCB =
+ [](const IRPosition &, const AbstractAttribute *,
+ bool &) -> std::optional<Value *> { return nullptr; };
+ for (const auto &It : AllocationInfos)
+ A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),
+ SCB);
+ for (const auto &It : DeallocationInfos)
+ A.registerSimplificationCallback(IRPosition::callsite_returned(*It.first),
+ SCB);
+ }
+
+ const std::string getAsStr(Attributor *A) const override {
+ unsigned NumH2SMallocs = 0, NumInvalidMallocs = 0;
+ for (const auto &It : AllocationInfos) {
+ if (It.second->Status == AllocationInfo::INVALID)
+ ++NumInvalidMallocs;
+ else
+ ++NumH2SMallocs;
+ }
+ return "[H2S] Mallocs Good/Bad: " + std::to_string(NumH2SMallocs) + "/" +
+ std::to_string(NumInvalidMallocs);
+ }
+
+ /// See AbstractAttribute::trackStatistics().
+ void trackStatistics() const override {
+ STATS_DECL(
+ MallocCalls, Function,
+ "Number of malloc/calloc/aligned_alloc calls converted to allocas");
+ for (const auto &It : AllocationInfos)
+ if (It.second->Status != AllocationInfo::INVALID)
+ ++BUILD_STAT_NAME(MallocCalls, Function);
+ }
+
+ bool isAssumedHeapToStack(const CallBase &CB) const override {
+ if (isValidState())
+ if (AllocationInfo *AI =
+ AllocationInfos.lookup(const_cast<CallBase *>(&CB)))
+ return AI->Status != AllocationInfo::INVALID;
+ return false;
+ }
+
+ bool isAssumedHeapToStackRemovedFree(CallBase &CB) const override {
+ if (!isValidState())
+ return false;
+
+ for (const auto &It : AllocationInfos) {
+ AllocationInfo &AI = *It.second;
+ if (AI.Status == AllocationInfo::INVALID)
+ continue;
+
+ if (AI.PotentialFreeCalls.count(&CB))
+ return true;
+ }
+
+ return false;
+ }
+
+ ChangeStatus manifest(Attributor &A) override {
+ assert(getState().isValidState() &&
+ "Attempted to manifest an invalid state!");
+
+ ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
+ Function *F = getAnchorScope();
+ const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
+
+ for (auto &It : AllocationInfos) {
+ AllocationInfo &AI = *It.second;
+ if (AI.Status == AllocationInfo::INVALID)
+ continue;
+
+ for (CallBase *FreeCall : AI.PotentialFreeCalls) {
+ LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
+ A.deleteAfterManifest(*FreeCall);
+ HasChanged = ChangeStatus::CHANGED;
+ }
+
+ LLVM_DEBUG(dbgs() << "H2S: Removing malloc-like call: " << *AI.CB
+ << "\n");
+
+ auto Remark = [&](OptimizationRemark OR) {
+ LibFunc IsAllocShared;
+ if (TLI->getLibFunc(*AI.CB, IsAllocShared))
+ if (IsAllocShared == LibFunc___kmpc_alloc_shared)
+ return OR << "Moving globalized variable to the stack.";
+ return OR << "Moving memory allocation from the heap to the stack.";
+ };
+ if (AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
+ A.emitRemark<OptimizationRemark>(AI.CB, "OMP110", Remark);
+ else
+ A.emitRemark<OptimizationRemark>(AI.CB, "HeapToStack", Remark);
+
+ const DataLayout &DL = A.getInfoCache().getDL();
+ Value *Size;
+ std::optional<APInt> SizeAPI = getSize(A, *this, AI);
+ if (SizeAPI) {
+ Size = ConstantInt::get(AI.CB->getContext(), *SizeAPI);
+ } else {
+ LLVMContext &Ctx = AI.CB->getContext();
+ ObjectSizeOpts Opts;
+ ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, Opts);
+ SizeOffsetValue SizeOffsetPair = Eval.compute(AI.CB);
+ assert(SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown() &&
+ cast<ConstantInt>(SizeOffsetPair.Offset)->isZero());
+ Size = SizeOffsetPair.Size;
+ }
+
+ BasicBlock::iterator IP = AI.MoveAllocaIntoEntry
+ ? F->getEntryBlock().begin()
+ : AI.CB->getIterator();
+
+ Align Alignment(1);
+ if (MaybeAlign RetAlign = AI.CB->getRetAlign())
+ Alignment = std::max(Alignment, *RetAlign);
+ if (Value *Align = getAllocAlignment(AI.CB, TLI)) {
+ std::optional<APInt> AlignmentAPI = getAPInt(A, *this, *Align);
+ assert(AlignmentAPI && AlignmentAPI->getZExtValue() > 0 &&
+ "Expected an alignment during manifest!");
+ Alignment =
+ std::max(Alignment, assumeAligned(AlignmentAPI->getZExtValue()));
+ }
+
+ // TODO: Hoist the alloca towards the function entry.
+ unsigned AS = DL.getAllocaAddrSpace();
+ Instruction *Alloca =
+ new AllocaInst(Type::getInt8Ty(F->getContext()), AS, Size, Alignment,
+ AI.CB->getName() + ".h2s", IP);
+
+ if (Alloca->getType() != AI.CB->getType())
+ Alloca = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
+ Alloca, AI.CB->getType(), "malloc_cast", AI.CB->getIterator());
+
+ auto *I8Ty = Type::getInt8Ty(F->getContext());
+ auto *InitVal = getInitialValueOfAllocation(AI.CB, TLI, I8Ty);
+ assert(InitVal &&
+ "Must be able to materialize initial memory state of allocation");
+
+ A.changeAfterManifest(IRPosition::inst(*AI.CB), *Alloca);
+
+ if (auto *II = dyn_cast<InvokeInst>(AI.CB)) {
+ auto *NBB = II->getNormalDest();
+ BranchInst::Create(NBB, AI.CB->getParent());
+ A.deleteAfterManifest(*AI.CB);
+ } else {
+ A.deleteAfterManifest(*AI.CB);
+ }
+
+ // Initialize the alloca with the same value as used by the allocation
+ // function. We can skip undef as the initial value of an alloc is
+ // undef, and the memset would simply end up being DSEd.
+ if (!isa<UndefValue>(InitVal)) {
+ IRBuilder<> Builder(Alloca->getNextNode());
+ // TODO: Use alignment above if align!=1
+ Builder.CreateMemSet(Alloca, InitVal, Size, std::nullopt);
+ }
+ HasChanged = ChangeStatus::CHANGED;
+ }
+
+ return HasChanged;
+ }
+
+ std::optional<APInt> getAPInt(Attributor &A, const AbstractAttribute &AA,
+ Value &V) {
+ bool UsedAssumedInformation = false;
+ std::optional<Constant *> SimpleV =
+ A.getAssumedConstant(V, AA, UsedAssumedInformation);
+ if (!SimpleV)
+ return APInt(64, 0);
+ if (auto *CI = dyn_cast_or_null<ConstantInt>(*SimpleV))
+ return CI->getValue();
+ return std::nullopt;
+ }
+
+ std::optional<APInt> getSize(Attributor &A, const AbstractAttribute &AA,
+ AllocationInfo &AI) {
+ auto Mapper = [&](const Value *V) -> const Value * {
+ bool UsedAssumedInformation = false;
+ if (std::optional<Constant *> SimpleV =
+ A.getAssumedConstant(*V, AA, UsedAssumedInformation))
+ if (*SimpleV)
+ return *SimpleV;
+ return V;
+ };
+
+ const Function *F = getAnchorScope();
+ const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
+ return getAllocSize(AI.CB, TLI, Mapper);
+ }
+
+ /// Collection of all malloc-like calls in a function with associated
+ /// information.
+ MapVector<CallBase *, AllocationInfo *> AllocationInfos;
+
+ /// Collection of all free-like calls in a function with associated
+ /// information.
+ MapVector<CallBase *, DeallocationInfo *> DeallocationInfos;
+
+ ChangeStatus updateImpl(Attributor &A) override;
+};
+
+ChangeStatus AAHeapToStackFunction::updateImpl(Attributor &A) {
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ const Function *F = getAnchorScope();
+ const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
+
+ const auto *LivenessAA =
+ A.getAAFor<AAIsDead>(*this, IRPosition::function(*F), DepClassTy::NONE);
+
+ MustBeExecutedContextExplorer *Explorer =
+ A.getInfoCache().getMustBeExecutedContextExplorer();
+
+ bool StackIsAccessibleByOtherThreads =
+ A.getInfoCache().stackIsAccessibleByOtherThreads();
+
+ LoopInfo *LI =
+ A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(*F);
+ std::optional<bool> MayContainIrreducibleControl;
+ auto IsInLoop = [&](BasicBlock &BB) {
+ if (&F->getEntryBlock() == &BB)
+ return false;
+ if (!MayContainIrreducibleControl.has_value())
+ MayContainIrreducibleControl = mayContainIrreducibleControl(*F, LI);
+ if (*MayContainIrreducibleControl)
+ return true;
+ if (!LI)
+ return true;
+ return LI->getLoopFor(&BB) != nullptr;
+ };
+
+ // Flag to ensure we update our deallocation information at most once per
+ // updateImpl call and only if we use the free check reasoning.
+ bool HasUpdatedFrees = false;
+
+ auto UpdateFrees = [&]() {
+ HasUpdatedFrees = true;
+
+ for (auto &It : DeallocationInfos) {
+ DeallocationInfo &DI = *It.second;
+ // For now we cannot use deallocations that have unknown inputs, skip
+ // them.
+ if (DI.MightFreeUnknownObjects)
+ continue;
+
+ // No need to analyze dead calls, ignore them instead.
+ bool UsedAssumedInformation = false;
+ if (A.isAssumedDead(*DI.CB, this, LivenessAA, UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ true))
+ continue;
+
+ // Use the non-optimistic version to get the freed object.
+ Value *Obj = getUnderlyingObject(DI.FreedOp);
+ if (!Obj) {
+ LLVM_DEBUG(dbgs() << "[H2S] Unknown underlying object for free!\n");
+ DI.MightFreeUnknownObjects = true;
+ continue;
+ }
+
+ // Free of null and undef can be ignored as no-ops (or UB in the latter
+ // case).
+ if (isa<ConstantPointerNull>(Obj) || isa<UndefValue>(Obj))
+ continue;
+
+ CallBase *ObjCB = dyn_cast<CallBase>(Obj);
+ if (!ObjCB) {
+ LLVM_DEBUG(dbgs() << "[H2S] Free of a non-call object: " << *Obj
+ << "\n");
+ DI.MightFreeUnknownObjects = true;
+ continue;
+ }
+
+ AllocationInfo *AI = AllocationInfos.lookup(ObjCB);
+ if (!AI) {
+ LLVM_DEBUG(dbgs() << "[H2S] Free of a non-allocation object: " << *Obj
+ << "\n");
+ DI.MightFreeUnknownObjects = true;
+ continue;
+ }
+
+ DI.PotentialAllocationCalls.insert(ObjCB);
+ }
+ };
+
+ auto FreeCheck = [&](AllocationInfo &AI) {
+ // If the stack is not accessible by other threads, the "must-free" logic
+ // doesn't apply as the pointer could be shared and needs to be places in
+ // "shareable" memory.
+ if (!StackIsAccessibleByOtherThreads) {
+ bool IsKnownNoSycn;
+ if (!AA::hasAssumedIRAttr<Attribute::NoSync>(
+ A, this, getIRPosition(), DepClassTy::OPTIONAL, IsKnownNoSycn)) {
+ LLVM_DEBUG(
+ dbgs() << "[H2S] found an escaping use, stack is not accessible by "
+ "other threads and function is not nosync:\n");
+ return false;
+ }
+ }
+ if (!HasUpdatedFrees)
+ UpdateFrees();
+
+ // TODO: Allow multi exit functions that have different free calls.
+ if (AI.PotentialFreeCalls.size() != 1) {
+ LLVM_DEBUG(dbgs() << "[H2S] did not find one free call but "
+ << AI.PotentialFreeCalls.size() << "\n");
+ return false;
+ }
+ CallBase *UniqueFree = *AI.PotentialFreeCalls.begin();
+ DeallocationInfo *DI = DeallocationInfos.lookup(UniqueFree);
+ if (!DI) {
+ LLVM_DEBUG(
+ dbgs() << "[H2S] unique free call was not known as deallocation call "
+ << *UniqueFree << "\n");
+ return false;
+ }
+ if (DI->MightFreeUnknownObjects) {
+ LLVM_DEBUG(
+ dbgs() << "[H2S] unique free call might free unknown allocations\n");
+ return false;
+ }
+ if (DI->PotentialAllocationCalls.empty())
+ return true;
+ if (DI->PotentialAllocationCalls.size() > 1) {
+ LLVM_DEBUG(dbgs() << "[H2S] unique free call might free "
+ << DI->PotentialAllocationCalls.size()
+ << " different allocations\n");
+ return false;
+ }
+ if (*DI->PotentialAllocationCalls.begin() != AI.CB) {
+ LLVM_DEBUG(
+ dbgs()
+ << "[H2S] unique free call not known to free this allocation but "
+ << **DI->PotentialAllocationCalls.begin() << "\n");
+ return false;
+ }
+
+ // __kmpc_alloc_shared and __kmpc_alloc_free are by construction matched.
+ if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared) {
+ Instruction *CtxI = isa<InvokeInst>(AI.CB) ? AI.CB : AI.CB->getNextNode();
+ if (!Explorer || !Explorer->findInContextOf(UniqueFree, CtxI)) {
+ LLVM_DEBUG(
+ dbgs()
+ << "[H2S] unique free call might not be executed with the allocation "
+ << *UniqueFree << "\n");
+ return false;
+ }
+ }
+ return true;
+ };
+
+ auto UsesCheck = [&](AllocationInfo &AI) {
+ bool ValidUsesOnly = true;
+
+ auto Pred = [&](const Use &U, bool &Follow) -> bool {
+ Instruction *UserI = cast<Instruction>(U.getUser());
+ if (isa<LoadInst>(UserI))
+ return true;
+ if (auto *SI = dyn_cast<StoreInst>(UserI)) {
+ if (SI->getValueOperand() == U.get()) {
+ LLVM_DEBUG(dbgs()
+ << "[H2S] escaping store to memory: " << *UserI << "\n");
+ ValidUsesOnly = false;
+ } else {
+ // A store into the malloc'ed memory is fine.
+ }
+ return true;
+ }
+ if (auto *CB = dyn_cast<CallBase>(UserI)) {
+ if (!CB->isArgOperand(&U) || CB->isLifetimeStartOrEnd())
+ return true;
+ if (DeallocationInfos.count(CB)) {
+ AI.PotentialFreeCalls.insert(CB);
+ return true;
+ }
+
+ unsigned ArgNo = CB->getArgOperandNo(&U);
+ auto CBIRP = IRPosition::callsite_argument(*CB, ArgNo);
+
+ bool IsKnownNoCapture;
+ bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoCapture);
+
+ // If a call site argument use is nofree, we are fine.
+ bool IsKnownNoFree;
+ bool IsAssumedNoFree = AA::hasAssumedIRAttr<Attribute::NoFree>(
+ A, this, CBIRP, DepClassTy::OPTIONAL, IsKnownNoFree);
+
+ if (!IsAssumedNoCapture ||
+ (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
+ !IsAssumedNoFree)) {
+ AI.HasPotentiallyFreeingUnknownUses |= !IsAssumedNoFree;
+
+ // Emit a missed remark if this is missed OpenMP globalization.
+ auto Remark = [&](OptimizationRemarkMissed ORM) {
+ return ORM
+ << "Could not move globalized variable to the stack. "
+ "Variable is potentially captured in call. Mark "
+ "parameter as `__attribute__((noescape))` to override.";
+ };
+
+ if (ValidUsesOnly &&
+ AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared)
+ A.emitRemark<OptimizationRemarkMissed>(CB, "OMP113", Remark);
+
+ LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
+ ValidUsesOnly = false;
+ }
+ return true;
+ }
+
+ if (isa<GetElementPtrInst>(UserI) || isa<BitCastInst>(UserI) ||
+ isa<PHINode>(UserI) || isa<SelectInst>(UserI)) {
+ Follow = true;
+ return true;
+ }
+ // Unknown user for which we can not track uses further (in a way that
+ // makes sense).
+ LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
+ ValidUsesOnly = false;
+ return true;
+ };
+ if (!A.checkForAllUses(Pred, *this, *AI.CB, /* CheckBBLivenessOnly */ false,
+ DepClassTy::OPTIONAL, /* IgnoreDroppableUses */ true,
+ [&](const Use &OldU, const Use &NewU) {
+ auto *SI = dyn_cast<StoreInst>(OldU.getUser());
+ return !SI || StackIsAccessibleByOtherThreads ||
+ AA::isAssumedThreadLocalObject(
+ A, *SI->getPointerOperand(), *this);
+ }))
+ return false;
+ return ValidUsesOnly;
+ };
+
+ // The actual update starts here. We look at all allocations and depending on
+ // their status perform the appropriate check(s).
+ for (auto &It : AllocationInfos) {
+ AllocationInfo &AI = *It.second;
+ if (AI.Status == AllocationInfo::INVALID)
+ continue;
+
+ if (Value *Align = getAllocAlignment(AI.CB, TLI)) {
+ std::optional<APInt> APAlign = getAPInt(A, *this, *Align);
+ if (!APAlign) {
+ // Can't generate an alloca which respects the required alignment
+ // on the allocation.
+ LLVM_DEBUG(dbgs() << "[H2S] Unknown allocation alignment: " << *AI.CB
+ << "\n");
+ AI.Status = AllocationInfo::INVALID;
+ Changed = ChangeStatus::CHANGED;
+ continue;
+ }
+ if (APAlign->ugt(llvm::Value::MaximumAlignment) ||
+ !APAlign->isPowerOf2()) {
+ LLVM_DEBUG(dbgs() << "[H2S] Invalid allocation alignment: " << APAlign
+ << "\n");
+ AI.Status = AllocationInfo::INVALID;
+ Changed = ChangeStatus::CHANGED;
+ continue;
+ }
+ }
+
+ std::optional<APInt> Size = getSize(A, *this, AI);
+ if (AI.LibraryFunctionId != LibFunc___kmpc_alloc_shared &&
+ MaxHeapToStackSize != -1) {
+ if (!Size || Size->ugt(MaxHeapToStackSize)) {
+ LLVM_DEBUG({
+ if (!Size)
+ dbgs() << "[H2S] Unknown allocation size: " << *AI.CB << "\n";
+ else
+ dbgs() << "[H2S] Allocation size too large: " << *AI.CB << " vs. "
+ << MaxHeapToStackSize << "\n";
+ });
+
+ AI.Status = AllocationInfo::INVALID;
+ Changed = ChangeStatus::CHANGED;
+ continue;
+ }
+ }
+
+ switch (AI.Status) {
+ case AllocationInfo::STACK_DUE_TO_USE:
+ if (UsesCheck(AI))
+ break;
+ AI.Status = AllocationInfo::STACK_DUE_TO_FREE;
+ [[fallthrough]];
+ case AllocationInfo::STACK_DUE_TO_FREE:
+ if (FreeCheck(AI))
+ break;
+ AI.Status = AllocationInfo::INVALID;
+ Changed = ChangeStatus::CHANGED;
+ break;
+ case AllocationInfo::INVALID:
+ llvm_unreachable("Invalid allocations should never reach this point!");
+ };
+
+ // Check if we still think we can move it into the entry block. If the
+ // alloca comes from a converted __kmpc_alloc_shared then we can usually
+ // ignore the potential compilations associated with loops.
+ bool IsGlobalizedLocal =
+ AI.LibraryFunctionId == LibFunc___kmpc_alloc_shared;
+ if (AI.MoveAllocaIntoEntry &&
+ (!Size.has_value() ||
+ (!IsGlobalizedLocal && IsInLoop(*AI.CB->getParent()))))
+ AI.MoveAllocaIntoEntry = false;
+ }
+
+ return Changed;
+}
+} // namespace
+
+/// ----------------------- Privatizable Pointers ------------------------------
+namespace {
+struct AAPrivatizablePtrImpl : public AAPrivatizablePtr {
+ AAPrivatizablePtrImpl(const IRPosition &IRP, Attributor &A)
+ : AAPrivatizablePtr(IRP, A), PrivatizableType(std::nullopt) {}
+
+ ChangeStatus indicatePessimisticFixpoint() override {
+ AAPrivatizablePtr::indicatePessimisticFixpoint();
+ PrivatizableType = nullptr;
+ return ChangeStatus::CHANGED;
+ }
+
+ /// Identify the type we can chose for a private copy of the underlying
+ /// argument. std::nullopt means it is not clear yet, nullptr means there is
+ /// none.
+ virtual std::optional<Type *> identifyPrivatizableType(Attributor &A) = 0;
+
+ /// Return a privatizable type that encloses both T0 and T1.
+ /// TODO: This is merely a stub for now as we should manage a mapping as well.
+ std::optional<Type *> combineTypes(std::optional<Type *> T0,
+ std::optional<Type *> T1) {
+ if (!T0)
+ return T1;
+ if (!T1)
+ return T0;
+ if (T0 == T1)
+ return T0;
+ return nullptr;
+ }
+
+ std::optional<Type *> getPrivatizableType() const override {
+ return PrivatizableType;
+ }
+
+ const std::string getAsStr(Attributor *A) const override {
+ return isAssumedPrivatizablePtr() ? "[priv]" : "[no-priv]";
+ }
+
+protected:
+ std::optional<Type *> PrivatizableType;
+};
+
+// TODO: Do this for call site arguments (probably also other values) as well.
+
+struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl {
+ AAPrivatizablePtrArgument(const IRPosition &IRP, Attributor &A)
+ : AAPrivatizablePtrImpl(IRP, A) {}
+
+ /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
+ std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
+ // If this is a byval argument and we know all the call sites (so we can
+ // rewrite them), there is no need to check them explicitly.
+ bool UsedAssumedInformation = false;
+ SmallVector<Attribute, 1> Attrs;
+ A.getAttrs(getIRPosition(), {Attribute::ByVal}, Attrs,
+ /* IgnoreSubsumingPositions */ true);
+ if (!Attrs.empty() &&
+ A.checkForAllCallSites([](AbstractCallSite ACS) { return true; }, *this,
+ true, UsedAssumedInformation))
+ return Attrs[0].getValueAsType();
+
+ std::optional<Type *> Ty;
+ unsigned ArgNo = getIRPosition().getCallSiteArgNo();
+
+ // Make sure the associated call site argument has the same type at all call
+ // sites and it is an allocation we know is safe to privatize, for now that
+ // means we only allow alloca instructions.
+ // TODO: We can additionally analyze the accesses in the callee to create
+ // the type from that information instead. That is a little more
+ // involved and will be done in a follow up patch.
+ auto CallSiteCheck = [&](AbstractCallSite ACS) {
+ IRPosition ACSArgPos = IRPosition::callsite_argument(ACS, ArgNo);
+ // Check if a coresponding argument was found or if it is one not
+ // associated (which can happen for callback calls).
+ if (ACSArgPos.getPositionKind() == IRPosition::IRP_INVALID)
+ return false;
+
+ // Check that all call sites agree on a type.
+ auto *PrivCSArgAA =
+ A.getAAFor<AAPrivatizablePtr>(*this, ACSArgPos, DepClassTy::REQUIRED);
+ if (!PrivCSArgAA)
+ return false;
+ std::optional<Type *> CSTy = PrivCSArgAA->getPrivatizableType();
+
+ LLVM_DEBUG({
+ dbgs() << "[AAPrivatizablePtr] ACSPos: " << ACSArgPos << ", CSTy: ";
+ if (CSTy && *CSTy)
+ (*CSTy)->print(dbgs());
+ else if (CSTy)
+ dbgs() << "<nullptr>";
+ else
+ dbgs() << "<none>";
+ });
+
+ Ty = combineTypes(Ty, CSTy);
+
+ LLVM_DEBUG({
+ dbgs() << " : New Type: ";
+ if (Ty && *Ty)
+ (*Ty)->print(dbgs());
+ else if (Ty)
+ dbgs() << "<nullptr>";
+ else
+ dbgs() << "<none>";
+ dbgs() << "\n";
+ });
+
+ return !Ty || *Ty;
+ };
+
+ if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
+ UsedAssumedInformation))
+ return nullptr;
+ return Ty;
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ PrivatizableType = identifyPrivatizableType(A);
+ if (!PrivatizableType)
+ return ChangeStatus::UNCHANGED;
+ if (!*PrivatizableType)
+ return indicatePessimisticFixpoint();
+
+ // The dependence is optional so we don't give up once we give up on the
+ // alignment.
+ A.getAAFor<AAAlign>(*this, IRPosition::value(getAssociatedValue()),
+ DepClassTy::OPTIONAL);
+
+ // Avoid arguments with padding for now.
+ if (!A.hasAttr(getIRPosition(), Attribute::ByVal) &&
+ !isDenselyPacked(*PrivatizableType, A.getInfoCache().getDL())) {
+ LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Padding detected\n");
+ return indicatePessimisticFixpoint();
+ }
+
+ // Collect the types that will replace the privatizable type in the function
+ // signature.
+ SmallVector<Type *, 16> ReplacementTypes;
+ identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
+
+ // Verify callee and caller agree on how the promoted argument would be
+ // passed.
+ Function &Fn = *getIRPosition().getAnchorScope();
+ const auto *TTI =
+ A.getInfoCache().getAnalysisResultForFunction<TargetIRAnalysis>(Fn);
+ if (!TTI) {
+ LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Missing TTI for function "
+ << Fn.getName() << "\n");
+ return indicatePessimisticFixpoint();
+ }
+
+ auto CallSiteCheck = [&](AbstractCallSite ACS) {
+ CallBase *CB = ACS.getInstruction();
+ return TTI->areTypesABICompatible(
+ CB->getCaller(),
+ dyn_cast_if_present<Function>(CB->getCalledOperand()),
+ ReplacementTypes);
+ };
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllCallSites(CallSiteCheck, *this, true,
+ UsedAssumedInformation)) {
+ LLVM_DEBUG(
+ dbgs() << "[AAPrivatizablePtr] ABI incompatibility detected for "
+ << Fn.getName() << "\n");
+ return indicatePessimisticFixpoint();
+ }
+
+ // Register a rewrite of the argument.
+ Argument *Arg = getAssociatedArgument();
+ if (!A.isValidFunctionSignatureRewrite(*Arg, ReplacementTypes)) {
+ LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Rewrite not valid\n");
+ return indicatePessimisticFixpoint();
+ }
+
+ unsigned ArgNo = Arg->getArgNo();
+
+ // Helper to check if for the given call site the associated argument is
+ // passed to a callback where the privatization would be different.
+ auto IsCompatiblePrivArgOfCallback = [&](CallBase &CB) {
+ SmallVector<const Use *, 4> CallbackUses;
+ AbstractCallSite::getCallbackUses(CB, CallbackUses);
+ for (const Use *U : CallbackUses) {
+ AbstractCallSite CBACS(U);
+ assert(CBACS && CBACS.isCallbackCall());
+ for (Argument &CBArg : CBACS.getCalledFunction()->args()) {
+ int CBArgNo = CBACS.getCallArgOperandNo(CBArg);
+
+ LLVM_DEBUG({
+ dbgs()
+ << "[AAPrivatizablePtr] Argument " << *Arg
+ << "check if can be privatized in the context of its parent ("
+ << Arg->getParent()->getName()
+ << ")\n[AAPrivatizablePtr] because it is an argument in a "
+ "callback ("
+ << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
+ << ")\n[AAPrivatizablePtr] " << CBArg << " : "
+ << CBACS.getCallArgOperand(CBArg) << " vs "
+ << CB.getArgOperand(ArgNo) << "\n"
+ << "[AAPrivatizablePtr] " << CBArg << " : "
+ << CBACS.getCallArgOperandNo(CBArg) << " vs " << ArgNo << "\n";
+ });
+
+ if (CBArgNo != int(ArgNo))
+ continue;
+ const auto *CBArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
+ *this, IRPosition::argument(CBArg), DepClassTy::REQUIRED);
+ if (CBArgPrivAA && CBArgPrivAA->isValidState()) {
+ auto CBArgPrivTy = CBArgPrivAA->getPrivatizableType();
+ if (!CBArgPrivTy)
+ continue;
+ if (*CBArgPrivTy == PrivatizableType)
+ continue;
+ }
+
+ LLVM_DEBUG({
+ dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
+ << " cannot be privatized in the context of its parent ("
+ << Arg->getParent()->getName()
+ << ")\n[AAPrivatizablePtr] because it is an argument in a "
+ "callback ("
+ << CBArgNo << "@" << CBACS.getCalledFunction()->getName()
+ << ").\n[AAPrivatizablePtr] for which the argument "
+ "privatization is not compatible.\n";
+ });
+ return false;
+ }
+ }
+ return true;
+ };
+
+ // Helper to check if for the given call site the associated argument is
+ // passed to a direct call where the privatization would be different.
+ auto IsCompatiblePrivArgOfDirectCS = [&](AbstractCallSite ACS) {
+ CallBase *DC = cast<CallBase>(ACS.getInstruction());
+ int DCArgNo = ACS.getCallArgOperandNo(ArgNo);
+ assert(DCArgNo >= 0 && unsigned(DCArgNo) < DC->arg_size() &&
+ "Expected a direct call operand for callback call operand");
+
+ Function *DCCallee =
+ dyn_cast_if_present<Function>(DC->getCalledOperand());
+ LLVM_DEBUG({
+ dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
+ << " check if be privatized in the context of its parent ("
+ << Arg->getParent()->getName()
+ << ")\n[AAPrivatizablePtr] because it is an argument in a "
+ "direct call of ("
+ << DCArgNo << "@" << DCCallee->getName() << ").\n";
+ });
+
+ if (unsigned(DCArgNo) < DCCallee->arg_size()) {
+ const auto *DCArgPrivAA = A.getAAFor<AAPrivatizablePtr>(
+ *this, IRPosition::argument(*DCCallee->getArg(DCArgNo)),
+ DepClassTy::REQUIRED);
+ if (DCArgPrivAA && DCArgPrivAA->isValidState()) {
+ auto DCArgPrivTy = DCArgPrivAA->getPrivatizableType();
+ if (!DCArgPrivTy)
+ return true;
+ if (*DCArgPrivTy == PrivatizableType)
+ return true;
+ }
+ }
+
+ LLVM_DEBUG({
+ dbgs() << "[AAPrivatizablePtr] Argument " << *Arg
+ << " cannot be privatized in the context of its parent ("
+ << Arg->getParent()->getName()
+ << ")\n[AAPrivatizablePtr] because it is an argument in a "
+ "direct call of ("
+ << ACS.getInstruction()->getCalledOperand()->getName()
+ << ").\n[AAPrivatizablePtr] for which the argument "
+ "privatization is not compatible.\n";
+ });
+ return false;
+ };
+
+ // Helper to check if the associated argument is used at the given abstract
+ // call site in a way that is incompatible with the privatization assumed
+ // here.
+ auto IsCompatiblePrivArgOfOtherCallSite = [&](AbstractCallSite ACS) {
+ if (ACS.isDirectCall())
+ return IsCompatiblePrivArgOfCallback(*ACS.getInstruction());
+ if (ACS.isCallbackCall())
+ return IsCompatiblePrivArgOfDirectCS(ACS);
+ return false;
+ };
+
+ if (!A.checkForAllCallSites(IsCompatiblePrivArgOfOtherCallSite, *this, true,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// Given a type to private \p PrivType, collect the constituates (which are
+ /// used) in \p ReplacementTypes.
+ static void
+ identifyReplacementTypes(Type *PrivType,
+ SmallVectorImpl<Type *> &ReplacementTypes) {
+ // TODO: For now we expand the privatization type to the fullest which can
+ // lead to dead arguments that need to be removed later.
+ assert(PrivType && "Expected privatizable type!");
+
+ // Traverse the type, extract constituate types on the outermost level.
+ if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
+ for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++)
+ ReplacementTypes.push_back(PrivStructType->getElementType(u));
+ } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
+ ReplacementTypes.append(PrivArrayType->getNumElements(),
+ PrivArrayType->getElementType());
+ } else {
+ ReplacementTypes.push_back(PrivType);
+ }
+ }
+
+ /// Initialize \p Base according to the type \p PrivType at position \p IP.
+ /// The values needed are taken from the arguments of \p F starting at
+ /// position \p ArgNo.
+ static void createInitialization(Type *PrivType, Value &Base, Function &F,
+ unsigned ArgNo, BasicBlock::iterator IP) {
+ assert(PrivType && "Expected privatizable type!");
+
+ IRBuilder<NoFolder> IRB(IP->getParent(), IP);
+ const DataLayout &DL = F.getParent()->getDataLayout();
+
+ // Traverse the type, build GEPs and stores.
+ if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
+ const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);
+ for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
+ Value *Ptr =
+ constructPointer(&Base, PrivStructLayout->getElementOffset(u), IRB);
+ new StoreInst(F.getArg(ArgNo + u), Ptr, IP);
+ }
+ } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
+ Type *PointeeTy = PrivArrayType->getElementType();
+ uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);
+ for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
+ Value *Ptr = constructPointer(&Base, u * PointeeTySize, IRB);
+ new StoreInst(F.getArg(ArgNo + u), Ptr, IP);
+ }
+ } else {
+ new StoreInst(F.getArg(ArgNo), &Base, IP);
+ }
+ }
+
+ /// Extract values from \p Base according to the type \p PrivType at the
+ /// call position \p ACS. The values are appended to \p ReplacementValues.
+ void createReplacementValues(Align Alignment, Type *PrivType,
+ AbstractCallSite ACS, Value *Base,
+ SmallVectorImpl<Value *> &ReplacementValues) {
+ assert(Base && "Expected base value!");
+ assert(PrivType && "Expected privatizable type!");
+ Instruction *IP = ACS.getInstruction();
+
+ IRBuilder<NoFolder> IRB(IP);
+ const DataLayout &DL = IP->getModule()->getDataLayout();
+
+ // Traverse the type, build GEPs and loads.
+ if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) {
+ const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType);
+ for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) {
+ Type *PointeeTy = PrivStructType->getElementType(u);
+ Value *Ptr =
+ constructPointer(Base, PrivStructLayout->getElementOffset(u), IRB);
+ LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
+ L->setAlignment(Alignment);
+ ReplacementValues.push_back(L);
+ }
+ } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) {
+ Type *PointeeTy = PrivArrayType->getElementType();
+ uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy);
+ for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) {
+ Value *Ptr = constructPointer(Base, u * PointeeTySize, IRB);
+ LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP->getIterator());
+ L->setAlignment(Alignment);
+ ReplacementValues.push_back(L);
+ }
+ } else {
+ LoadInst *L = new LoadInst(PrivType, Base, "", IP->getIterator());
+ L->setAlignment(Alignment);
+ ReplacementValues.push_back(L);
+ }
+ }
+
+ /// See AbstractAttribute::manifest(...)
+ ChangeStatus manifest(Attributor &A) override {
+ if (!PrivatizableType)
+ return ChangeStatus::UNCHANGED;
+ assert(*PrivatizableType && "Expected privatizable type!");
+
+ // Collect all tail calls in the function as we cannot allow new allocas to
+ // escape into tail recursion.
+ // TODO: Be smarter about new allocas escaping into tail calls.
+ SmallVector<CallInst *, 16> TailCalls;
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllInstructions(
+ [&](Instruction &I) {
+ CallInst &CI = cast<CallInst>(I);
+ if (CI.isTailCall())
+ TailCalls.push_back(&CI);
+ return true;
+ },
+ *this, {Instruction::Call}, UsedAssumedInformation))
+ return ChangeStatus::UNCHANGED;
+
+ Argument *Arg = getAssociatedArgument();
+ // Query AAAlign attribute for alignment of associated argument to
+ // determine the best alignment of loads.
+ const auto *AlignAA =
+ A.getAAFor<AAAlign>(*this, IRPosition::value(*Arg), DepClassTy::NONE);
+
+ // Callback to repair the associated function. A new alloca is placed at the
+ // beginning and initialized with the values passed through arguments. The
+ // new alloca replaces the use of the old pointer argument.
+ Attributor::ArgumentReplacementInfo::CalleeRepairCBTy FnRepairCB =
+ [=](const Attributor::ArgumentReplacementInfo &ARI,
+ Function &ReplacementFn, Function::arg_iterator ArgIt) {
+ BasicBlock &EntryBB = ReplacementFn.getEntryBlock();
+ BasicBlock::iterator IP = EntryBB.getFirstInsertionPt();
+ const DataLayout &DL = IP->getModule()->getDataLayout();
+ unsigned AS = DL.getAllocaAddrSpace();
+ Instruction *AI = new AllocaInst(*PrivatizableType, AS,
+ Arg->getName() + ".priv", IP);
+ createInitialization(*PrivatizableType, *AI, ReplacementFn,
+ ArgIt->getArgNo(), IP);
+
+ if (AI->getType() != Arg->getType())
+ AI = BitCastInst::CreatePointerBitCastOrAddrSpaceCast(
+ AI, Arg->getType(), "", IP);
+ Arg->replaceAllUsesWith(AI);
+
+ for (CallInst *CI : TailCalls)
+ CI->setTailCall(false);
+ };
+
+ // Callback to repair a call site of the associated function. The elements
+ // of the privatizable type are loaded prior to the call and passed to the
+ // new function version.
+ Attributor::ArgumentReplacementInfo::ACSRepairCBTy ACSRepairCB =
+ [=](const Attributor::ArgumentReplacementInfo &ARI,
+ AbstractCallSite ACS, SmallVectorImpl<Value *> &NewArgOperands) {
+ // When no alignment is specified for the load instruction,
+ // natural alignment is assumed.
+ createReplacementValues(
+ AlignAA ? AlignAA->getAssumedAlign() : Align(0),
+ *PrivatizableType, ACS,
+ ACS.getCallArgOperand(ARI.getReplacedArg().getArgNo()),
+ NewArgOperands);
+ };
+
+ // Collect the types that will replace the privatizable type in the function
+ // signature.
+ SmallVector<Type *, 16> ReplacementTypes;
+ identifyReplacementTypes(*PrivatizableType, ReplacementTypes);
+
+ // Register a rewrite of the argument.
+ if (A.registerFunctionSignatureRewrite(*Arg, ReplacementTypes,
+ std::move(FnRepairCB),
+ std::move(ACSRepairCB)))
+ return ChangeStatus::CHANGED;
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_ARG_ATTR(privatizable_ptr);
+ }
+};
+
+struct AAPrivatizablePtrFloating : public AAPrivatizablePtrImpl {
+ AAPrivatizablePtrFloating(const IRPosition &IRP, Attributor &A)
+ : AAPrivatizablePtrImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ // TODO: We can privatize more than arguments.
+ indicatePessimisticFixpoint();
+ }
+
+ ChangeStatus updateImpl(Attributor &A) override {
+ llvm_unreachable("AAPrivatizablePtr(Floating|Returned|CallSiteReturned)::"
+ "updateImpl will not be called");
+ }
+
+ /// See AAPrivatizablePtrImpl::identifyPrivatizableType(...)
+ std::optional<Type *> identifyPrivatizableType(Attributor &A) override {
+ Value *Obj = getUnderlyingObject(&getAssociatedValue());
+ if (!Obj) {
+ LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] No underlying object found!\n");
+ return nullptr;
+ }
+
+ if (auto *AI = dyn_cast<AllocaInst>(Obj))
+ if (auto *CI = dyn_cast<ConstantInt>(AI->getArraySize()))
+ if (CI->isOne())
+ return AI->getAllocatedType();
+ if (auto *Arg = dyn_cast<Argument>(Obj)) {
+ auto *PrivArgAA = A.getAAFor<AAPrivatizablePtr>(
+ *this, IRPosition::argument(*Arg), DepClassTy::REQUIRED);
+ if (PrivArgAA && PrivArgAA->isAssumedPrivatizablePtr())
+ return PrivArgAA->getPrivatizableType();
+ }
+
+ LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] Underlying object neither valid "
+ "alloca nor privatizable argument: "
+ << *Obj << "!\n");
+ return nullptr;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(privatizable_ptr);
+ }
+};
+
+struct AAPrivatizablePtrCallSiteArgument final
+ : public AAPrivatizablePtrFloating {
+ AAPrivatizablePtrCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAPrivatizablePtrFloating(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ if (A.hasAttr(getIRPosition(), Attribute::ByVal))
+ indicateOptimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ PrivatizableType = identifyPrivatizableType(A);
+ if (!PrivatizableType)
+ return ChangeStatus::UNCHANGED;
+ if (!*PrivatizableType)
+ return indicatePessimisticFixpoint();
+
+ const IRPosition &IRP = getIRPosition();
+ bool IsKnownNoCapture;
+ bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, IRP, DepClassTy::REQUIRED, IsKnownNoCapture);
+ if (!IsAssumedNoCapture) {
+ LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might be captured!\n");
+ return indicatePessimisticFixpoint();
+ }
+
+ bool IsKnownNoAlias;
+ if (!AA::hasAssumedIRAttr<Attribute::NoAlias>(
+ A, this, IRP, DepClassTy::REQUIRED, IsKnownNoAlias)) {
+ LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer might alias!\n");
+ return indicatePessimisticFixpoint();
+ }
+
+ bool IsKnown;
+ if (!AA::isAssumedReadOnly(A, IRP, *this, IsKnown)) {
+ LLVM_DEBUG(dbgs() << "[AAPrivatizablePtr] pointer is written!\n");
+ return indicatePessimisticFixpoint();
+ }
+
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSARG_ATTR(privatizable_ptr);
+ }
+};
+
+struct AAPrivatizablePtrCallSiteReturned final
+ : public AAPrivatizablePtrFloating {
+ AAPrivatizablePtrCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AAPrivatizablePtrFloating(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ // TODO: We can privatize more than arguments.
+ indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSRET_ATTR(privatizable_ptr);
+ }
+};
+
+struct AAPrivatizablePtrReturned final : public AAPrivatizablePtrFloating {
+ AAPrivatizablePtrReturned(const IRPosition &IRP, Attributor &A)
+ : AAPrivatizablePtrFloating(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ // TODO: We can privatize more than arguments.
+ indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FNRET_ATTR(privatizable_ptr);
+ }
+};
+} // namespace
+
+/// -------------------- Memory Behavior Attributes ----------------------------
+/// Includes read-none, read-only, and write-only.
+/// ----------------------------------------------------------------------------
+namespace {
+struct AAMemoryBehaviorImpl : public AAMemoryBehavior {
+ AAMemoryBehaviorImpl(const IRPosition &IRP, Attributor &A)
+ : AAMemoryBehavior(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ intersectAssumedBits(BEST_STATE);
+ getKnownStateFromValue(A, getIRPosition(), getState());
+ AAMemoryBehavior::initialize(A);
+ }
+
+ /// Return the memory behavior information encoded in the IR for \p IRP.
+ static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
+ BitIntegerState &State,
+ bool IgnoreSubsumingPositions = false) {
+ SmallVector<Attribute, 2> Attrs;
+ A.getAttrs(IRP, AttrKinds, Attrs, IgnoreSubsumingPositions);
+ for (const Attribute &Attr : Attrs) {
+ switch (Attr.getKindAsEnum()) {
+ case Attribute::ReadNone:
+ State.addKnownBits(NO_ACCESSES);
+ break;
+ case Attribute::ReadOnly:
+ State.addKnownBits(NO_WRITES);
+ break;
+ case Attribute::WriteOnly:
+ State.addKnownBits(NO_READS);
+ break;
+ default:
+ llvm_unreachable("Unexpected attribute!");
+ }
+ }
+
+ if (auto *I = dyn_cast<Instruction>(&IRP.getAnchorValue())) {
+ if (!I->mayReadFromMemory())
+ State.addKnownBits(NO_READS);
+ if (!I->mayWriteToMemory())
+ State.addKnownBits(NO_WRITES);
+ }
+ }
+
+ /// See AbstractAttribute::getDeducedAttributes(...).
+ void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
+ SmallVectorImpl<Attribute> &Attrs) const override {
+ assert(Attrs.size() == 0);
+ if (isAssumedReadNone())
+ Attrs.push_back(Attribute::get(Ctx, Attribute::ReadNone));
+ else if (isAssumedReadOnly())
+ Attrs.push_back(Attribute::get(Ctx, Attribute::ReadOnly));
+ else if (isAssumedWriteOnly())
+ Attrs.push_back(Attribute::get(Ctx, Attribute::WriteOnly));
+ assert(Attrs.size() <= 1);
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ const IRPosition &IRP = getIRPosition();
+
+ if (A.hasAttr(IRP, Attribute::ReadNone,
+ /* IgnoreSubsumingPositions */ true))
+ return ChangeStatus::UNCHANGED;
+
+ // Check if we would improve the existing attributes first.
+ SmallVector<Attribute, 4> DeducedAttrs;
+ getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);
+ if (llvm::all_of(DeducedAttrs, [&](const Attribute &Attr) {
+ return A.hasAttr(IRP, Attr.getKindAsEnum(),
+ /* IgnoreSubsumingPositions */ true);
+ }))
+ return ChangeStatus::UNCHANGED;
+
+ // Clear existing attributes.
+ A.removeAttrs(IRP, AttrKinds);
+ // Clear conflicting writable attribute.
+ if (isAssumedReadOnly())
+ A.removeAttrs(IRP, Attribute::Writable);
+
+ // Use the generic manifest method.
+ return IRAttribute::manifest(A);
+ }
+
+ /// See AbstractState::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ if (isAssumedReadNone())
+ return "readnone";
+ if (isAssumedReadOnly())
+ return "readonly";
+ if (isAssumedWriteOnly())
+ return "writeonly";
+ return "may-read/write";
+ }
+
+ /// The set of IR attributes AAMemoryBehavior deals with.
+ static const Attribute::AttrKind AttrKinds[3];
+};
+
+const Attribute::AttrKind AAMemoryBehaviorImpl::AttrKinds[] = {
+ Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly};
+
+/// Memory behavior attribute for a floating value.
+struct AAMemoryBehaviorFloating : AAMemoryBehaviorImpl {
+ AAMemoryBehaviorFloating(const IRPosition &IRP, Attributor &A)
+ : AAMemoryBehaviorImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override;
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ if (isAssumedReadNone())
+ STATS_DECLTRACK_FLOATING_ATTR(readnone)
+ else if (isAssumedReadOnly())
+ STATS_DECLTRACK_FLOATING_ATTR(readonly)
+ else if (isAssumedWriteOnly())
+ STATS_DECLTRACK_FLOATING_ATTR(writeonly)
+ }
+
+private:
+ /// Return true if users of \p UserI might access the underlying
+ /// variable/location described by \p U and should therefore be analyzed.
+ bool followUsersOfUseIn(Attributor &A, const Use &U,
+ const Instruction *UserI);
+
+ /// Update the state according to the effect of use \p U in \p UserI.
+ void analyzeUseIn(Attributor &A, const Use &U, const Instruction *UserI);
+};
+
+/// Memory behavior attribute for function argument.
+struct AAMemoryBehaviorArgument : AAMemoryBehaviorFloating {
+ AAMemoryBehaviorArgument(const IRPosition &IRP, Attributor &A)
+ : AAMemoryBehaviorFloating(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ intersectAssumedBits(BEST_STATE);
+ const IRPosition &IRP = getIRPosition();
+ // TODO: Make IgnoreSubsumingPositions a property of an IRAttribute so we
+ // can query it when we use has/getAttr. That would allow us to reuse the
+ // initialize of the base class here.
+ bool HasByVal = A.hasAttr(IRP, {Attribute::ByVal},
+ /* IgnoreSubsumingPositions */ true);
+ getKnownStateFromValue(A, IRP, getState(),
+ /* IgnoreSubsumingPositions */ HasByVal);
+ }
+
+ ChangeStatus manifest(Attributor &A) override {
+ // TODO: Pointer arguments are not supported on vectors of pointers yet.
+ if (!getAssociatedValue().getType()->isPointerTy())
+ return ChangeStatus::UNCHANGED;
+
+ // TODO: From readattrs.ll: "inalloca parameters are always
+ // considered written"
+ if (A.hasAttr(getIRPosition(),
+ {Attribute::InAlloca, Attribute::Preallocated})) {
+ removeKnownBits(NO_WRITES);
+ removeAssumedBits(NO_WRITES);
+ }
+ A.removeAttrs(getIRPosition(), AttrKinds);
+ return AAMemoryBehaviorFloating::manifest(A);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ if (isAssumedReadNone())
+ STATS_DECLTRACK_ARG_ATTR(readnone)
+ else if (isAssumedReadOnly())
+ STATS_DECLTRACK_ARG_ATTR(readonly)
+ else if (isAssumedWriteOnly())
+ STATS_DECLTRACK_ARG_ATTR(writeonly)
+ }
+};
+
+struct AAMemoryBehaviorCallSiteArgument final : AAMemoryBehaviorArgument {
+ AAMemoryBehaviorCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAMemoryBehaviorArgument(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ // If we don't have an associated attribute this is either a variadic call
+ // or an indirect call, either way, nothing to do here.
+ Argument *Arg = getAssociatedArgument();
+ if (!Arg) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+ if (Arg->hasByValAttr()) {
+ addKnownBits(NO_WRITES);
+ removeKnownBits(NO_READS);
+ removeAssumedBits(NO_READS);
+ }
+ AAMemoryBehaviorArgument::initialize(A);
+ if (getAssociatedFunction()->isDeclaration())
+ indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // TODO: Once we have call site specific value information we can provide
+ // call site specific liveness liveness information and then it makes
+ // sense to specialize attributes for call sites arguments instead of
+ // redirecting requests to the callee argument.
+ Argument *Arg = getAssociatedArgument();
+ const IRPosition &ArgPos = IRPosition::argument(*Arg);
+ auto *ArgAA =
+ A.getAAFor<AAMemoryBehavior>(*this, ArgPos, DepClassTy::REQUIRED);
+ if (!ArgAA)
+ return indicatePessimisticFixpoint();
+ return clampStateAndIndicateChange(getState(), ArgAA->getState());
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ if (isAssumedReadNone())
+ STATS_DECLTRACK_CSARG_ATTR(readnone)
+ else if (isAssumedReadOnly())
+ STATS_DECLTRACK_CSARG_ATTR(readonly)
+ else if (isAssumedWriteOnly())
+ STATS_DECLTRACK_CSARG_ATTR(writeonly)
+ }
+};
+
+/// Memory behavior attribute for a call site return position.
+struct AAMemoryBehaviorCallSiteReturned final : AAMemoryBehaviorFloating {
+ AAMemoryBehaviorCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AAMemoryBehaviorFloating(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ AAMemoryBehaviorImpl::initialize(A);
+ }
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ // We do not annotate returned values.
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {}
+};
+
+/// An AA to represent the memory behavior function attributes.
+struct AAMemoryBehaviorFunction final : public AAMemoryBehaviorImpl {
+ AAMemoryBehaviorFunction(const IRPosition &IRP, Attributor &A)
+ : AAMemoryBehaviorImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(Attributor &A).
+ ChangeStatus updateImpl(Attributor &A) override;
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ // TODO: It would be better to merge this with AAMemoryLocation, so that
+ // we could determine read/write per location. This would also have the
+ // benefit of only one place trying to manifest the memory attribute.
+ Function &F = cast<Function>(getAnchorValue());
+ MemoryEffects ME = MemoryEffects::unknown();
+ if (isAssumedReadNone())
+ ME = MemoryEffects::none();
+ else if (isAssumedReadOnly())
+ ME = MemoryEffects::readOnly();
+ else if (isAssumedWriteOnly())
+ ME = MemoryEffects::writeOnly();
+
+ A.removeAttrs(getIRPosition(), AttrKinds);
+ // Clear conflicting writable attribute.
+ if (ME.onlyReadsMemory())
+ for (Argument &Arg : F.args())
+ A.removeAttrs(IRPosition::argument(Arg), Attribute::Writable);
+ return A.manifestAttrs(getIRPosition(),
+ Attribute::getWithMemoryEffects(F.getContext(), ME));
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ if (isAssumedReadNone())
+ STATS_DECLTRACK_FN_ATTR(readnone)
+ else if (isAssumedReadOnly())
+ STATS_DECLTRACK_FN_ATTR(readonly)
+ else if (isAssumedWriteOnly())
+ STATS_DECLTRACK_FN_ATTR(writeonly)
+ }
+};
+
+/// AAMemoryBehavior attribute for call sites.
+struct AAMemoryBehaviorCallSite final
+ : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl> {
+ AAMemoryBehaviorCallSite(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AAMemoryBehavior, AAMemoryBehaviorImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ // TODO: Deduplicate this with AAMemoryBehaviorFunction.
+ CallBase &CB = cast<CallBase>(getAnchorValue());
+ MemoryEffects ME = MemoryEffects::unknown();
+ if (isAssumedReadNone())
+ ME = MemoryEffects::none();
+ else if (isAssumedReadOnly())
+ ME = MemoryEffects::readOnly();
+ else if (isAssumedWriteOnly())
+ ME = MemoryEffects::writeOnly();
+
+ A.removeAttrs(getIRPosition(), AttrKinds);
+ // Clear conflicting writable attribute.
+ if (ME.onlyReadsMemory())
+ for (Use &U : CB.args())
+ A.removeAttrs(IRPosition::callsite_argument(CB, U.getOperandNo()),
+ Attribute::Writable);
+ return A.manifestAttrs(
+ getIRPosition(), Attribute::getWithMemoryEffects(CB.getContext(), ME));
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ if (isAssumedReadNone())
+ STATS_DECLTRACK_CS_ATTR(readnone)
+ else if (isAssumedReadOnly())
+ STATS_DECLTRACK_CS_ATTR(readonly)
+ else if (isAssumedWriteOnly())
+ STATS_DECLTRACK_CS_ATTR(writeonly)
+ }
+};
+
+ChangeStatus AAMemoryBehaviorFunction::updateImpl(Attributor &A) {
+
+ // The current assumed state used to determine a change.
+ auto AssumedState = getAssumed();
+
+ auto CheckRWInst = [&](Instruction &I) {
+ // If the instruction has an own memory behavior state, use it to restrict
+ // the local state. No further analysis is required as the other memory
+ // state is as optimistic as it gets.
+ if (const auto *CB = dyn_cast<CallBase>(&I)) {
+ const auto *MemBehaviorAA = A.getAAFor<AAMemoryBehavior>(
+ *this, IRPosition::callsite_function(*CB), DepClassTy::REQUIRED);
+ if (MemBehaviorAA) {
+ intersectAssumedBits(MemBehaviorAA->getAssumed());
+ return !isAtFixpoint();
+ }
+ }
+
+ // Remove access kind modifiers if necessary.
+ if (I.mayReadFromMemory())
+ removeAssumedBits(NO_READS);
+ if (I.mayWriteToMemory())
+ removeAssumedBits(NO_WRITES);
+ return !isAtFixpoint();
+ };
+
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
+ : ChangeStatus::UNCHANGED;
+}
+
+ChangeStatus AAMemoryBehaviorFloating::updateImpl(Attributor &A) {
+
+ const IRPosition &IRP = getIRPosition();
+ const IRPosition &FnPos = IRPosition::function_scope(IRP);
+ AAMemoryBehavior::StateType &S = getState();
+
+ // First, check the function scope. We take the known information and we avoid
+ // work if the assumed information implies the current assumed information for
+ // this attribute. This is a valid for all but byval arguments.
+ Argument *Arg = IRP.getAssociatedArgument();
+ AAMemoryBehavior::base_t FnMemAssumedState =
+ AAMemoryBehavior::StateType::getWorstState();
+ if (!Arg || !Arg->hasByValAttr()) {
+ const auto *FnMemAA =
+ A.getAAFor<AAMemoryBehavior>(*this, FnPos, DepClassTy::OPTIONAL);
+ if (FnMemAA) {
+ FnMemAssumedState = FnMemAA->getAssumed();
+ S.addKnownBits(FnMemAA->getKnown());
+ if ((S.getAssumed() & FnMemAA->getAssumed()) == S.getAssumed())
+ return ChangeStatus::UNCHANGED;
+ }
+ }
+
+ // The current assumed state used to determine a change.
+ auto AssumedState = S.getAssumed();
+
+ // Make sure the value is not captured (except through "return"), if
+ // it is, any information derived would be irrelevant anyway as we cannot
+ // check the potential aliases introduced by the capture. However, no need
+ // to fall back to anythign less optimistic than the function state.
+ bool IsKnownNoCapture;
+ const AANoCapture *ArgNoCaptureAA = nullptr;
+ bool IsAssumedNoCapture = AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture, false,
+ &ArgNoCaptureAA);
+
+ if (!IsAssumedNoCapture &&
+ (!ArgNoCaptureAA || !ArgNoCaptureAA->isAssumedNoCaptureMaybeReturned())) {
+ S.intersectAssumedBits(FnMemAssumedState);
+ return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
+ : ChangeStatus::UNCHANGED;
+ }
+
+ // Visit and expand uses until all are analyzed or a fixpoint is reached.
+ auto UsePred = [&](const Use &U, bool &Follow) -> bool {
+ Instruction *UserI = cast<Instruction>(U.getUser());
+ LLVM_DEBUG(dbgs() << "[AAMemoryBehavior] Use: " << *U << " in " << *UserI
+ << " \n");
+
+ // Droppable users, e.g., llvm::assume does not actually perform any action.
+ if (UserI->isDroppable())
+ return true;
+
+ // Check if the users of UserI should also be visited.
+ Follow = followUsersOfUseIn(A, U, UserI);
+
+ // If UserI might touch memory we analyze the use in detail.
+ if (UserI->mayReadOrWriteMemory())
+ analyzeUseIn(A, U, UserI);
+
+ return !isAtFixpoint();
+ };
+
+ if (!A.checkForAllUses(UsePred, *this, getAssociatedValue()))
+ return indicatePessimisticFixpoint();
+
+ return (AssumedState != getAssumed()) ? ChangeStatus::CHANGED
+ : ChangeStatus::UNCHANGED;
+}
+
+bool AAMemoryBehaviorFloating::followUsersOfUseIn(Attributor &A, const Use &U,
+ const Instruction *UserI) {
+ // The loaded value is unrelated to the pointer argument, no need to
+ // follow the users of the load.
+ if (isa<LoadInst>(UserI) || isa<ReturnInst>(UserI))
+ return false;
+
+ // By default we follow all uses assuming UserI might leak information on U,
+ // we have special handling for call sites operands though.
+ const auto *CB = dyn_cast<CallBase>(UserI);
+ if (!CB || !CB->isArgOperand(&U))
+ return true;
+
+ // If the use is a call argument known not to be captured, the users of
+ // the call do not need to be visited because they have to be unrelated to
+ // the input. Note that this check is not trivial even though we disallow
+ // general capturing of the underlying argument. The reason is that the
+ // call might the argument "through return", which we allow and for which we
+ // need to check call users.
+ if (U.get()->getType()->isPointerTy()) {
+ unsigned ArgNo = CB->getArgOperandNo(&U);
+ bool IsKnownNoCapture;
+ return !AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, IRPosition::callsite_argument(*CB, ArgNo),
+ DepClassTy::OPTIONAL, IsKnownNoCapture);
+ }
+
+ return true;
+}
+
+void AAMemoryBehaviorFloating::analyzeUseIn(Attributor &A, const Use &U,
+ const Instruction *UserI) {
+ assert(UserI->mayReadOrWriteMemory());
+
+ switch (UserI->getOpcode()) {
+ default:
+ // TODO: Handle all atomics and other side-effect operations we know of.
+ break;
+ case Instruction::Load:
+ // Loads cause the NO_READS property to disappear.
+ removeAssumedBits(NO_READS);
+ return;
+
+ case Instruction::Store:
+ // Stores cause the NO_WRITES property to disappear if the use is the
+ // pointer operand. Note that while capturing was taken care of somewhere
+ // else we need to deal with stores of the value that is not looked through.
+ if (cast<StoreInst>(UserI)->getPointerOperand() == U.get())
+ removeAssumedBits(NO_WRITES);
+ else
+ indicatePessimisticFixpoint();
+ return;
+
+ case Instruction::Call:
+ case Instruction::CallBr:
+ case Instruction::Invoke: {
+ // For call sites we look at the argument memory behavior attribute (this
+ // could be recursive!) in order to restrict our own state.
+ const auto *CB = cast<CallBase>(UserI);
+
+ // Give up on operand bundles.
+ if (CB->isBundleOperand(&U)) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+
+ // Calling a function does read the function pointer, maybe write it if the
+ // function is self-modifying.
+ if (CB->isCallee(&U)) {
+ removeAssumedBits(NO_READS);
+ break;
+ }
+
+ // Adjust the possible access behavior based on the information on the
+ // argument.
+ IRPosition Pos;
+ if (U.get()->getType()->isPointerTy())
+ Pos = IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U));
+ else
+ Pos = IRPosition::callsite_function(*CB);
+ const auto *MemBehaviorAA =
+ A.getAAFor<AAMemoryBehavior>(*this, Pos, DepClassTy::OPTIONAL);
+ if (!MemBehaviorAA)
+ break;
+ // "assumed" has at most the same bits as the MemBehaviorAA assumed
+ // and at least "known".
+ intersectAssumedBits(MemBehaviorAA->getAssumed());
+ return;
+ }
+ };
+
+ // Generally, look at the "may-properties" and adjust the assumed state if we
+ // did not trigger special handling before.
+ if (UserI->mayReadFromMemory())
+ removeAssumedBits(NO_READS);
+ if (UserI->mayWriteToMemory())
+ removeAssumedBits(NO_WRITES);
+}
+} // namespace
+
+/// -------------------- Memory Locations Attributes ---------------------------
+/// Includes read-none, argmemonly, inaccessiblememonly,
+/// inaccessiblememorargmemonly
+/// ----------------------------------------------------------------------------
+
+std::string AAMemoryLocation::getMemoryLocationsAsStr(
+ AAMemoryLocation::MemoryLocationsKind MLK) {
+ if (0 == (MLK & AAMemoryLocation::NO_LOCATIONS))
+ return "all memory";
+ if (MLK == AAMemoryLocation::NO_LOCATIONS)
+ return "no memory";
+ std::string S = "memory:";
+ if (0 == (MLK & AAMemoryLocation::NO_LOCAL_MEM))
+ S += "stack,";
+ if (0 == (MLK & AAMemoryLocation::NO_CONST_MEM))
+ S += "constant,";
+ if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_INTERNAL_MEM))
+ S += "internal global,";
+ if (0 == (MLK & AAMemoryLocation::NO_GLOBAL_EXTERNAL_MEM))
+ S += "external global,";
+ if (0 == (MLK & AAMemoryLocation::NO_ARGUMENT_MEM))
+ S += "argument,";
+ if (0 == (MLK & AAMemoryLocation::NO_INACCESSIBLE_MEM))
+ S += "inaccessible,";
+ if (0 == (MLK & AAMemoryLocation::NO_MALLOCED_MEM))
+ S += "malloced,";
+ if (0 == (MLK & AAMemoryLocation::NO_UNKOWN_MEM))
+ S += "unknown,";
+ S.pop_back();
+ return S;
+}
+
+namespace {
+struct AAMemoryLocationImpl : public AAMemoryLocation {
+
+ AAMemoryLocationImpl(const IRPosition &IRP, Attributor &A)
+ : AAMemoryLocation(IRP, A), Allocator(A.Allocator) {
+ AccessKind2Accesses.fill(nullptr);
+ }
+
+ ~AAMemoryLocationImpl() {
+ // The AccessSets are allocated via a BumpPtrAllocator, we call
+ // the destructor manually.
+ for (AccessSet *AS : AccessKind2Accesses)
+ if (AS)
+ AS->~AccessSet();
+ }
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ intersectAssumedBits(BEST_STATE);
+ getKnownStateFromValue(A, getIRPosition(), getState());
+ AAMemoryLocation::initialize(A);
+ }
+
+ /// Return the memory behavior information encoded in the IR for \p IRP.
+ static void getKnownStateFromValue(Attributor &A, const IRPosition &IRP,
+ BitIntegerState &State,
+ bool IgnoreSubsumingPositions = false) {
+ // For internal functions we ignore `argmemonly` and
+ // `inaccessiblememorargmemonly` as we might break it via interprocedural
+ // constant propagation. It is unclear if this is the best way but it is
+ // unlikely this will cause real performance problems. If we are deriving
+ // attributes for the anchor function we even remove the attribute in
+ // addition to ignoring it.
+ // TODO: A better way to handle this would be to add ~NO_GLOBAL_MEM /
+ // MemoryEffects::Other as a possible location.
+ bool UseArgMemOnly = true;
+ Function *AnchorFn = IRP.getAnchorScope();
+ if (AnchorFn && A.isRunOn(*AnchorFn))
+ UseArgMemOnly = !AnchorFn->hasLocalLinkage();
+
+ SmallVector<Attribute, 2> Attrs;
+ A.getAttrs(IRP, {Attribute::Memory}, Attrs, IgnoreSubsumingPositions);
+ for (const Attribute &Attr : Attrs) {
+ // TODO: We can map MemoryEffects to Attributor locations more precisely.
+ MemoryEffects ME = Attr.getMemoryEffects();
+ if (ME.doesNotAccessMemory()) {
+ State.addKnownBits(NO_LOCAL_MEM | NO_CONST_MEM);
+ continue;
+ }
+ if (ME.onlyAccessesInaccessibleMem()) {
+ State.addKnownBits(inverseLocation(NO_INACCESSIBLE_MEM, true, true));
+ continue;
+ }
+ if (ME.onlyAccessesArgPointees()) {
+ if (UseArgMemOnly)
+ State.addKnownBits(inverseLocation(NO_ARGUMENT_MEM, true, true));
+ else {
+ // Remove location information, only keep read/write info.
+ ME = MemoryEffects(ME.getModRef());
+ A.manifestAttrs(IRP,
+ Attribute::getWithMemoryEffects(
+ IRP.getAnchorValue().getContext(), ME),
+ /*ForceReplace*/ true);
+ }
+ continue;
+ }
+ if (ME.onlyAccessesInaccessibleOrArgMem()) {
+ if (UseArgMemOnly)
+ State.addKnownBits(inverseLocation(
+ NO_INACCESSIBLE_MEM | NO_ARGUMENT_MEM, true, true));
+ else {
+ // Remove location information, only keep read/write info.
+ ME = MemoryEffects(ME.getModRef());
+ A.manifestAttrs(IRP,
+ Attribute::getWithMemoryEffects(
+ IRP.getAnchorValue().getContext(), ME),
+ /*ForceReplace*/ true);
+ }
+ continue;
+ }
+ }
+ }
+
+ /// See AbstractAttribute::getDeducedAttributes(...).
+ void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
+ SmallVectorImpl<Attribute> &Attrs) const override {
+ // TODO: We can map Attributor locations to MemoryEffects more precisely.
+ assert(Attrs.size() == 0);
+ if (getIRPosition().getPositionKind() == IRPosition::IRP_FUNCTION) {
+ if (isAssumedReadNone())
+ Attrs.push_back(
+ Attribute::getWithMemoryEffects(Ctx, MemoryEffects::none()));
+ else if (isAssumedInaccessibleMemOnly())
+ Attrs.push_back(Attribute::getWithMemoryEffects(
+ Ctx, MemoryEffects::inaccessibleMemOnly()));
+ else if (isAssumedArgMemOnly())
+ Attrs.push_back(
+ Attribute::getWithMemoryEffects(Ctx, MemoryEffects::argMemOnly()));
+ else if (isAssumedInaccessibleOrArgMemOnly())
+ Attrs.push_back(Attribute::getWithMemoryEffects(
+ Ctx, MemoryEffects::inaccessibleOrArgMemOnly()));
+ }
+ assert(Attrs.size() <= 1);
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ // TODO: If AAMemoryLocation and AAMemoryBehavior are merged, we could
+ // provide per-location modref information here.
+ const IRPosition &IRP = getIRPosition();
+
+ SmallVector<Attribute, 1> DeducedAttrs;
+ getDeducedAttributes(A, IRP.getAnchorValue().getContext(), DeducedAttrs);
+ if (DeducedAttrs.size() != 1)
+ return ChangeStatus::UNCHANGED;
+ MemoryEffects ME = DeducedAttrs[0].getMemoryEffects();
+
+ return A.manifestAttrs(IRP, Attribute::getWithMemoryEffects(
+ IRP.getAnchorValue().getContext(), ME));
+ }
+
+ /// See AAMemoryLocation::checkForAllAccessesToMemoryKind(...).
+ bool checkForAllAccessesToMemoryKind(
+ function_ref<bool(const Instruction *, const Value *, AccessKind,
+ MemoryLocationsKind)>
+ Pred,
+ MemoryLocationsKind RequestedMLK) const override {
+ if (!isValidState())
+ return false;
+
+ MemoryLocationsKind AssumedMLK = getAssumedNotAccessedLocation();
+ if (AssumedMLK == NO_LOCATIONS)
+ return true;
+
+ unsigned Idx = 0;
+ for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS;
+ CurMLK *= 2, ++Idx) {
+ if (CurMLK & RequestedMLK)
+ continue;
+
+ if (const AccessSet *Accesses = AccessKind2Accesses[Idx])
+ for (const AccessInfo &AI : *Accesses)
+ if (!Pred(AI.I, AI.Ptr, AI.Kind, CurMLK))
+ return false;
+ }
+
+ return true;
+ }
+
+ ChangeStatus indicatePessimisticFixpoint() override {
+ // If we give up and indicate a pessimistic fixpoint this instruction will
+ // become an access for all potential access kinds:
+ // TODO: Add pointers for argmemonly and globals to improve the results of
+ // checkForAllAccessesToMemoryKind.
+ bool Changed = false;
+ MemoryLocationsKind KnownMLK = getKnown();
+ Instruction *I = dyn_cast<Instruction>(&getAssociatedValue());
+ for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2)
+ if (!(CurMLK & KnownMLK))
+ updateStateAndAccessesMap(getState(), CurMLK, I, nullptr, Changed,
+ getAccessKindFromInst(I));
+ return AAMemoryLocation::indicatePessimisticFixpoint();
+ }
+
+protected:
+ /// Helper struct to tie together an instruction that has a read or write
+ /// effect with the pointer it accesses (if any).
+ struct AccessInfo {
+
+ /// The instruction that caused the access.
+ const Instruction *I;
+
+ /// The base pointer that is accessed, or null if unknown.
+ const Value *Ptr;
+
+ /// The kind of access (read/write/read+write).
+ AccessKind Kind;
+
+ bool operator==(const AccessInfo &RHS) const {
+ return I == RHS.I && Ptr == RHS.Ptr && Kind == RHS.Kind;
+ }
+ bool operator()(const AccessInfo &LHS, const AccessInfo &RHS) const {
+ if (LHS.I != RHS.I)
+ return LHS.I < RHS.I;
+ if (LHS.Ptr != RHS.Ptr)
+ return LHS.Ptr < RHS.Ptr;
+ if (LHS.Kind != RHS.Kind)
+ return LHS.Kind < RHS.Kind;
+ return false;
+ }
+ };
+
+ /// Mapping from *single* memory location kinds, e.g., LOCAL_MEM with the
+ /// value of NO_LOCAL_MEM, to the accesses encountered for this memory kind.
+ using AccessSet = SmallSet<AccessInfo, 2, AccessInfo>;
+ std::array<AccessSet *, llvm::CTLog2<VALID_STATE>()> AccessKind2Accesses;
+
+ /// Categorize the pointer arguments of CB that might access memory in
+ /// AccessedLoc and update the state and access map accordingly.
+ void
+ categorizeArgumentPointerLocations(Attributor &A, CallBase &CB,
+ AAMemoryLocation::StateType &AccessedLocs,
+ bool &Changed);
+
+ /// Return the kind(s) of location that may be accessed by \p V.
+ AAMemoryLocation::MemoryLocationsKind
+ categorizeAccessedLocations(Attributor &A, Instruction &I, bool &Changed);
+
+ /// Return the access kind as determined by \p I.
+ AccessKind getAccessKindFromInst(const Instruction *I) {
+ AccessKind AK = READ_WRITE;
+ if (I) {
+ AK = I->mayReadFromMemory() ? READ : NONE;
+ AK = AccessKind(AK | (I->mayWriteToMemory() ? WRITE : NONE));
+ }
+ return AK;
+ }
+
+ /// Update the state \p State and the AccessKind2Accesses given that \p I is
+ /// an access of kind \p AK to a \p MLK memory location with the access
+ /// pointer \p Ptr.
+ void updateStateAndAccessesMap(AAMemoryLocation::StateType &State,
+ MemoryLocationsKind MLK, const Instruction *I,
+ const Value *Ptr, bool &Changed,
+ AccessKind AK = READ_WRITE) {
+
+ assert(isPowerOf2_32(MLK) && "Expected a single location set!");
+ auto *&Accesses = AccessKind2Accesses[llvm::Log2_32(MLK)];
+ if (!Accesses)
+ Accesses = new (Allocator) AccessSet();
+ Changed |= Accesses->insert(AccessInfo{I, Ptr, AK}).second;
+ if (MLK == NO_UNKOWN_MEM)
+ MLK = NO_LOCATIONS;
+ State.removeAssumedBits(MLK);
+ }
+
+ /// Determine the underlying locations kinds for \p Ptr, e.g., globals or
+ /// arguments, and update the state and access map accordingly.
+ void categorizePtrValue(Attributor &A, const Instruction &I, const Value &Ptr,
+ AAMemoryLocation::StateType &State, bool &Changed,
+ unsigned AccessAS = 0);
+
+ /// Used to allocate access sets.
+ BumpPtrAllocator &Allocator;
+};
+
+void AAMemoryLocationImpl::categorizePtrValue(
+ Attributor &A, const Instruction &I, const Value &Ptr,
+ AAMemoryLocation::StateType &State, bool &Changed, unsigned AccessAS) {
+ LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize pointer locations for "
+ << Ptr << " ["
+ << getMemoryLocationsAsStr(State.getAssumed()) << "]\n");
+
+ auto Pred = [&](Value &Obj) {
+ unsigned ObjectAS = Obj.getType()->getPointerAddressSpace();
+ // TODO: recognize the TBAA used for constant accesses.
+ MemoryLocationsKind MLK = NO_LOCATIONS;
+
+ // Filter accesses to constant (GPU) memory if we have an AS at the access
+ // site or the object is known to actually have the associated AS.
+ if ((AccessAS == (unsigned)AA::GPUAddressSpace::Constant ||
+ (ObjectAS == (unsigned)AA::GPUAddressSpace::Constant &&
+ isIdentifiedObject(&Obj))) &&
+ AA::isGPU(*I.getModule()))
+ return true;
+
+ if (isa<UndefValue>(&Obj))
+ return true;
+ if (isa<Argument>(&Obj)) {
+ // TODO: For now we do not treat byval arguments as local copies performed
+ // on the call edge, though, we should. To make that happen we need to
+ // teach various passes, e.g., DSE, about the copy effect of a byval. That
+ // would also allow us to mark functions only accessing byval arguments as
+ // readnone again, arguably their accesses have no effect outside of the
+ // function, like accesses to allocas.
+ MLK = NO_ARGUMENT_MEM;
+ } else if (auto *GV = dyn_cast<GlobalValue>(&Obj)) {
+ // Reading constant memory is not treated as a read "effect" by the
+ // function attr pass so we won't neither. Constants defined by TBAA are
+ // similar. (We know we do not write it because it is constant.)
+ if (auto *GVar = dyn_cast<GlobalVariable>(GV))
+ if (GVar->isConstant())
+ return true;
+
+ if (GV->hasLocalLinkage())
+ MLK = NO_GLOBAL_INTERNAL_MEM;
+ else
+ MLK = NO_GLOBAL_EXTERNAL_MEM;
+ } else if (isa<ConstantPointerNull>(&Obj) &&
+ (!NullPointerIsDefined(getAssociatedFunction(), AccessAS) ||
+ !NullPointerIsDefined(getAssociatedFunction(), ObjectAS))) {
+ return true;
+ } else if (isa<AllocaInst>(&Obj)) {
+ MLK = NO_LOCAL_MEM;
+ } else if (const auto *CB = dyn_cast<CallBase>(&Obj)) {
+ bool IsKnownNoAlias;
+ if (AA::hasAssumedIRAttr<Attribute::NoAlias>(
+ A, this, IRPosition::callsite_returned(*CB), DepClassTy::OPTIONAL,
+ IsKnownNoAlias))
+ MLK = NO_MALLOCED_MEM;
+ else
+ MLK = NO_UNKOWN_MEM;
+ } else {
+ MLK = NO_UNKOWN_MEM;
+ }
+
+ assert(MLK != NO_LOCATIONS && "No location specified!");
+ LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Ptr value can be categorized: "
+ << Obj << " -> " << getMemoryLocationsAsStr(MLK) << "\n");
+ updateStateAndAccessesMap(State, MLK, &I, &Obj, Changed,
+ getAccessKindFromInst(&I));
+
+ return true;
+ };
+
+ const auto *AA = A.getAAFor<AAUnderlyingObjects>(
+ *this, IRPosition::value(Ptr), DepClassTy::OPTIONAL);
+ if (!AA || !AA->forallUnderlyingObjects(Pred, AA::Intraprocedural)) {
+ LLVM_DEBUG(
+ dbgs() << "[AAMemoryLocation] Pointer locations not categorized\n");
+ updateStateAndAccessesMap(State, NO_UNKOWN_MEM, &I, nullptr, Changed,
+ getAccessKindFromInst(&I));
+ return;
+ }
+
+ LLVM_DEBUG(
+ dbgs() << "[AAMemoryLocation] Accessed locations with pointer locations: "
+ << getMemoryLocationsAsStr(State.getAssumed()) << "\n");
+}
+
+void AAMemoryLocationImpl::categorizeArgumentPointerLocations(
+ Attributor &A, CallBase &CB, AAMemoryLocation::StateType &AccessedLocs,
+ bool &Changed) {
+ for (unsigned ArgNo = 0, E = CB.arg_size(); ArgNo < E; ++ArgNo) {
+
+ // Skip non-pointer arguments.
+ const Value *ArgOp = CB.getArgOperand(ArgNo);
+ if (!ArgOp->getType()->isPtrOrPtrVectorTy())
+ continue;
+
+ // Skip readnone arguments.
+ const IRPosition &ArgOpIRP = IRPosition::callsite_argument(CB, ArgNo);
+ const auto *ArgOpMemLocationAA =
+ A.getAAFor<AAMemoryBehavior>(*this, ArgOpIRP, DepClassTy::OPTIONAL);
+
+ if (ArgOpMemLocationAA && ArgOpMemLocationAA->isAssumedReadNone())
+ continue;
+
+ // Categorize potentially accessed pointer arguments as if there was an
+ // access instruction with them as pointer.
+ categorizePtrValue(A, CB, *ArgOp, AccessedLocs, Changed);
+ }
+}
+
+AAMemoryLocation::MemoryLocationsKind
+AAMemoryLocationImpl::categorizeAccessedLocations(Attributor &A, Instruction &I,
+ bool &Changed) {
+ LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize accessed locations for "
+ << I << "\n");
+
+ AAMemoryLocation::StateType AccessedLocs;
+ AccessedLocs.intersectAssumedBits(NO_LOCATIONS);
+
+ if (auto *CB = dyn_cast<CallBase>(&I)) {
+
+ // First check if we assume any memory is access is visible.
+ const auto *CBMemLocationAA = A.getAAFor<AAMemoryLocation>(
+ *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);
+ LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Categorize call site: " << I
+ << " [" << CBMemLocationAA << "]\n");
+ if (!CBMemLocationAA) {
+ updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr,
+ Changed, getAccessKindFromInst(&I));
+ return NO_UNKOWN_MEM;
+ }
+
+ if (CBMemLocationAA->isAssumedReadNone())
+ return NO_LOCATIONS;
+
+ if (CBMemLocationAA->isAssumedInaccessibleMemOnly()) {
+ updateStateAndAccessesMap(AccessedLocs, NO_INACCESSIBLE_MEM, &I, nullptr,
+ Changed, getAccessKindFromInst(&I));
+ return AccessedLocs.getAssumed();
+ }
+
+ uint32_t CBAssumedNotAccessedLocs =
+ CBMemLocationAA->getAssumedNotAccessedLocation();
+
+ // Set the argmemonly and global bit as we handle them separately below.
+ uint32_t CBAssumedNotAccessedLocsNoArgMem =
+ CBAssumedNotAccessedLocs | NO_ARGUMENT_MEM | NO_GLOBAL_MEM;
+
+ for (MemoryLocationsKind CurMLK = 1; CurMLK < NO_LOCATIONS; CurMLK *= 2) {
+ if (CBAssumedNotAccessedLocsNoArgMem & CurMLK)
+ continue;
+ updateStateAndAccessesMap(AccessedLocs, CurMLK, &I, nullptr, Changed,
+ getAccessKindFromInst(&I));
+ }
+
+ // Now handle global memory if it might be accessed. This is slightly tricky
+ // as NO_GLOBAL_MEM has multiple bits set.
+ bool HasGlobalAccesses = ((~CBAssumedNotAccessedLocs) & NO_GLOBAL_MEM);
+ if (HasGlobalAccesses) {
+ auto AccessPred = [&](const Instruction *, const Value *Ptr,
+ AccessKind Kind, MemoryLocationsKind MLK) {
+ updateStateAndAccessesMap(AccessedLocs, MLK, &I, Ptr, Changed,
+ getAccessKindFromInst(&I));
+ return true;
+ };
+ if (!CBMemLocationAA->checkForAllAccessesToMemoryKind(
+ AccessPred, inverseLocation(NO_GLOBAL_MEM, false, false)))
+ return AccessedLocs.getWorstState();
+ }
+
+ LLVM_DEBUG(
+ dbgs() << "[AAMemoryLocation] Accessed state before argument handling: "
+ << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
+
+ // Now handle argument memory if it might be accessed.
+ bool HasArgAccesses = ((~CBAssumedNotAccessedLocs) & NO_ARGUMENT_MEM);
+ if (HasArgAccesses)
+ categorizeArgumentPointerLocations(A, *CB, AccessedLocs, Changed);
+
+ LLVM_DEBUG(
+ dbgs() << "[AAMemoryLocation] Accessed state after argument handling: "
+ << getMemoryLocationsAsStr(AccessedLocs.getAssumed()) << "\n");
+
+ return AccessedLocs.getAssumed();
+ }
+
+ if (const Value *Ptr = getPointerOperand(&I, /* AllowVolatile */ true)) {
+ LLVM_DEBUG(
+ dbgs() << "[AAMemoryLocation] Categorize memory access with pointer: "
+ << I << " [" << *Ptr << "]\n");
+ categorizePtrValue(A, I, *Ptr, AccessedLocs, Changed,
+ Ptr->getType()->getPointerAddressSpace());
+ return AccessedLocs.getAssumed();
+ }
+
+ LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Failed to categorize instruction: "
+ << I << "\n");
+ updateStateAndAccessesMap(AccessedLocs, NO_UNKOWN_MEM, &I, nullptr, Changed,
+ getAccessKindFromInst(&I));
+ return AccessedLocs.getAssumed();
+}
+
+/// An AA to represent the memory behavior function attributes.
+struct AAMemoryLocationFunction final : public AAMemoryLocationImpl {
+ AAMemoryLocationFunction(const IRPosition &IRP, Attributor &A)
+ : AAMemoryLocationImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(Attributor &A).
+ ChangeStatus updateImpl(Attributor &A) override {
+
+ const auto *MemBehaviorAA =
+ A.getAAFor<AAMemoryBehavior>(*this, getIRPosition(), DepClassTy::NONE);
+ if (MemBehaviorAA && MemBehaviorAA->isAssumedReadNone()) {
+ if (MemBehaviorAA->isKnownReadNone())
+ return indicateOptimisticFixpoint();
+ assert(isAssumedReadNone() &&
+ "AAMemoryLocation was not read-none but AAMemoryBehavior was!");
+ A.recordDependence(*MemBehaviorAA, *this, DepClassTy::OPTIONAL);
+ return ChangeStatus::UNCHANGED;
+ }
+
+ // The current assumed state used to determine a change.
+ auto AssumedState = getAssumed();
+ bool Changed = false;
+
+ auto CheckRWInst = [&](Instruction &I) {
+ MemoryLocationsKind MLK = categorizeAccessedLocations(A, I, Changed);
+ LLVM_DEBUG(dbgs() << "[AAMemoryLocation] Accessed locations for " << I
+ << ": " << getMemoryLocationsAsStr(MLK) << "\n");
+ removeAssumedBits(inverseLocation(MLK, false, false));
+ // Stop once only the valid bit set in the *not assumed location*, thus
+ // once we don't actually exclude any memory locations in the state.
+ return getAssumedNotAccessedLocation() != VALID_STATE;
+ };
+
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllReadWriteInstructions(CheckRWInst, *this,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ Changed |= AssumedState != getAssumed();
+ return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ if (isAssumedReadNone())
+ STATS_DECLTRACK_FN_ATTR(readnone)
+ else if (isAssumedArgMemOnly())
+ STATS_DECLTRACK_FN_ATTR(argmemonly)
+ else if (isAssumedInaccessibleMemOnly())
+ STATS_DECLTRACK_FN_ATTR(inaccessiblememonly)
+ else if (isAssumedInaccessibleOrArgMemOnly())
+ STATS_DECLTRACK_FN_ATTR(inaccessiblememorargmemonly)
+ }
+};
+
+/// AAMemoryLocation attribute for call sites.
+struct AAMemoryLocationCallSite final : AAMemoryLocationImpl {
+ AAMemoryLocationCallSite(const IRPosition &IRP, Attributor &A)
+ : AAMemoryLocationImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ // TODO: Once we have call site specific value information we can provide
+ // call site specific liveness liveness information and then it makes
+ // sense to specialize attributes for call sites arguments instead of
+ // redirecting requests to the callee argument.
+ Function *F = getAssociatedFunction();
+ const IRPosition &FnPos = IRPosition::function(*F);
+ auto *FnAA =
+ A.getAAFor<AAMemoryLocation>(*this, FnPos, DepClassTy::REQUIRED);
+ if (!FnAA)
+ return indicatePessimisticFixpoint();
+ bool Changed = false;
+ auto AccessPred = [&](const Instruction *I, const Value *Ptr,
+ AccessKind Kind, MemoryLocationsKind MLK) {
+ updateStateAndAccessesMap(getState(), MLK, I, Ptr, Changed,
+ getAccessKindFromInst(I));
+ return true;
+ };
+ if (!FnAA->checkForAllAccessesToMemoryKind(AccessPred, ALL_LOCATIONS))
+ return indicatePessimisticFixpoint();
+ return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ if (isAssumedReadNone())
+ STATS_DECLTRACK_CS_ATTR(readnone)
+ }
+};
+} // namespace
+
+/// ------------------ denormal-fp-math Attribute -------------------------
+
+namespace {
+struct AADenormalFPMathImpl : public AADenormalFPMath {
+ AADenormalFPMathImpl(const IRPosition &IRP, Attributor &A)
+ : AADenormalFPMath(IRP, A) {}
+
+ const std::string getAsStr(Attributor *A) const override {
+ std::string Str("AADenormalFPMath[");
+ raw_string_ostream OS(Str);
+
+ DenormalState Known = getKnown();
+ if (Known.Mode.isValid())
+ OS << "denormal-fp-math=" << Known.Mode;
+ else
+ OS << "invalid";
+
+ if (Known.ModeF32.isValid())
+ OS << " denormal-fp-math-f32=" << Known.ModeF32;
+ OS << ']';
+ return OS.str();
+ }
+};
+
+struct AADenormalFPMathFunction final : AADenormalFPMathImpl {
+ AADenormalFPMathFunction(const IRPosition &IRP, Attributor &A)
+ : AADenormalFPMathImpl(IRP, A) {}
+
+ void initialize(Attributor &A) override {
+ const Function *F = getAnchorScope();
+ DenormalMode Mode = F->getDenormalModeRaw();
+ DenormalMode ModeF32 = F->getDenormalModeF32Raw();
+
+ // TODO: Handling this here prevents handling the case where a callee has a
+ // fixed denormal-fp-math with dynamic denormal-fp-math-f32, but called from
+ // a function with a fully fixed mode.
+ if (ModeF32 == DenormalMode::getInvalid())
+ ModeF32 = Mode;
+ Known = DenormalState{Mode, ModeF32};
+ if (isModeFixed())
+ indicateFixpoint();
+ }
+
+ ChangeStatus updateImpl(Attributor &A) override {
+ ChangeStatus Change = ChangeStatus::UNCHANGED;
+
+ auto CheckCallSite = [=, &Change, &A](AbstractCallSite CS) {
+ Function *Caller = CS.getInstruction()->getFunction();
+ LLVM_DEBUG(dbgs() << "[AADenormalFPMath] Call " << Caller->getName()
+ << "->" << getAssociatedFunction()->getName() << '\n');
+
+ const auto *CallerInfo = A.getAAFor<AADenormalFPMath>(
+ *this, IRPosition::function(*Caller), DepClassTy::REQUIRED);
+ if (!CallerInfo)
+ return false;
+
+ Change = Change | clampStateAndIndicateChange(this->getState(),
+ CallerInfo->getState());
+ return true;
+ };
+
+ bool AllCallSitesKnown = true;
+ if (!A.checkForAllCallSites(CheckCallSite, *this, true, AllCallSitesKnown))
+ return indicatePessimisticFixpoint();
+
+ if (Change == ChangeStatus::CHANGED && isModeFixed())
+ indicateFixpoint();
+ return Change;
+ }
+
+ ChangeStatus manifest(Attributor &A) override {
+ LLVMContext &Ctx = getAssociatedFunction()->getContext();
+
+ SmallVector<Attribute, 2> AttrToAdd;
+ SmallVector<StringRef, 2> AttrToRemove;
+ if (Known.Mode == DenormalMode::getDefault()) {
+ AttrToRemove.push_back("denormal-fp-math");
+ } else {
+ AttrToAdd.push_back(
+ Attribute::get(Ctx, "denormal-fp-math", Known.Mode.str()));
+ }
+
+ if (Known.ModeF32 != Known.Mode) {
+ AttrToAdd.push_back(
+ Attribute::get(Ctx, "denormal-fp-math-f32", Known.ModeF32.str()));
+ } else {
+ AttrToRemove.push_back("denormal-fp-math-f32");
+ }
+
+ auto &IRP = getIRPosition();
+
+ // TODO: There should be a combined add and remove API.
+ return A.removeAttrs(IRP, AttrToRemove) |
+ A.manifestAttrs(IRP, AttrToAdd, /*ForceReplace=*/true);
+ }
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FN_ATTR(denormal_fp_math)
+ }
+};
+} // namespace
+
+/// ------------------ Value Constant Range Attribute -------------------------
+
+namespace {
+struct AAValueConstantRangeImpl : AAValueConstantRange {
+ using StateType = IntegerRangeState;
+ AAValueConstantRangeImpl(const IRPosition &IRP, Attributor &A)
+ : AAValueConstantRange(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(..).
+ void initialize(Attributor &A) override {
+ if (A.hasSimplificationCallback(getIRPosition())) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+
+ // Intersect a range given by SCEV.
+ intersectKnown(getConstantRangeFromSCEV(A, getCtxI()));
+
+ // Intersect a range given by LVI.
+ intersectKnown(getConstantRangeFromLVI(A, getCtxI()));
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ std::string Str;
+ llvm::raw_string_ostream OS(Str);
+ OS << "range(" << getBitWidth() << ")<";
+ getKnown().print(OS);
+ OS << " / ";
+ getAssumed().print(OS);
+ OS << ">";
+ return OS.str();
+ }
+
+ /// Helper function to get a SCEV expr for the associated value at program
+ /// point \p I.
+ const SCEV *getSCEV(Attributor &A, const Instruction *I = nullptr) const {
+ if (!getAnchorScope())
+ return nullptr;
+
+ ScalarEvolution *SE =
+ A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
+ *getAnchorScope());
+
+ LoopInfo *LI = A.getInfoCache().getAnalysisResultForFunction<LoopAnalysis>(
+ *getAnchorScope());
+
+ if (!SE || !LI)
+ return nullptr;
+
+ const SCEV *S = SE->getSCEV(&getAssociatedValue());
+ if (!I)
+ return S;
+
+ return SE->getSCEVAtScope(S, LI->getLoopFor(I->getParent()));
+ }
+
+ /// Helper function to get a range from SCEV for the associated value at
+ /// program point \p I.
+ ConstantRange getConstantRangeFromSCEV(Attributor &A,
+ const Instruction *I = nullptr) const {
+ if (!getAnchorScope())
+ return getWorstState(getBitWidth());
+
+ ScalarEvolution *SE =
+ A.getInfoCache().getAnalysisResultForFunction<ScalarEvolutionAnalysis>(
+ *getAnchorScope());
+
+ const SCEV *S = getSCEV(A, I);
+ if (!SE || !S)
+ return getWorstState(getBitWidth());
+
+ return SE->getUnsignedRange(S);
+ }
+
+ /// Helper function to get a range from LVI for the associated value at
+ /// program point \p I.
+ ConstantRange
+ getConstantRangeFromLVI(Attributor &A,
+ const Instruction *CtxI = nullptr) const {
+ if (!getAnchorScope())
+ return getWorstState(getBitWidth());
+
+ LazyValueInfo *LVI =
+ A.getInfoCache().getAnalysisResultForFunction<LazyValueAnalysis>(
+ *getAnchorScope());
+
+ if (!LVI || !CtxI)
+ return getWorstState(getBitWidth());
+ return LVI->getConstantRange(&getAssociatedValue(),
+ const_cast<Instruction *>(CtxI),
+ /*UndefAllowed*/ false);
+ }
+
+ /// Return true if \p CtxI is valid for querying outside analyses.
+ /// This basically makes sure we do not ask intra-procedural analysis
+ /// about a context in the wrong function or a context that violates
+ /// dominance assumptions they might have. The \p AllowAACtxI flag indicates
+ /// if the original context of this AA is OK or should be considered invalid.
+ bool isValidCtxInstructionForOutsideAnalysis(Attributor &A,
+ const Instruction *CtxI,
+ bool AllowAACtxI) const {
+ if (!CtxI || (!AllowAACtxI && CtxI == getCtxI()))
+ return false;
+
+ // Our context might be in a different function, neither intra-procedural
+ // analysis (ScalarEvolution nor LazyValueInfo) can handle that.
+ if (!AA::isValidInScope(getAssociatedValue(), CtxI->getFunction()))
+ return false;
+
+ // If the context is not dominated by the value there are paths to the
+ // context that do not define the value. This cannot be handled by
+ // LazyValueInfo so we need to bail.
+ if (auto *I = dyn_cast<Instruction>(&getAssociatedValue())) {
+ InformationCache &InfoCache = A.getInfoCache();
+ const DominatorTree *DT =
+ InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(
+ *I->getFunction());
+ return DT && DT->dominates(I, CtxI);
+ }
+
+ return true;
+ }
+
+ /// See AAValueConstantRange::getKnownConstantRange(..).
+ ConstantRange
+ getKnownConstantRange(Attributor &A,
+ const Instruction *CtxI = nullptr) const override {
+ if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
+ /* AllowAACtxI */ false))
+ return getKnown();
+
+ ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
+ ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);
+ return getKnown().intersectWith(SCEVR).intersectWith(LVIR);
+ }
+
+ /// See AAValueConstantRange::getAssumedConstantRange(..).
+ ConstantRange
+ getAssumedConstantRange(Attributor &A,
+ const Instruction *CtxI = nullptr) const override {
+ // TODO: Make SCEV use Attributor assumption.
+ // We may be able to bound a variable range via assumptions in
+ // Attributor. ex.) If x is assumed to be in [1, 3] and y is known to
+ // evolve to x^2 + x, then we can say that y is in [2, 12].
+ if (!isValidCtxInstructionForOutsideAnalysis(A, CtxI,
+ /* AllowAACtxI */ false))
+ return getAssumed();
+
+ ConstantRange LVIR = getConstantRangeFromLVI(A, CtxI);
+ ConstantRange SCEVR = getConstantRangeFromSCEV(A, CtxI);
+ return getAssumed().intersectWith(SCEVR).intersectWith(LVIR);
+ }
+
+ /// Helper function to create MDNode for range metadata.
+ static MDNode *
+ getMDNodeForConstantRange(Type *Ty, LLVMContext &Ctx,
+ const ConstantRange &AssumedConstantRange) {
+ Metadata *LowAndHigh[] = {ConstantAsMetadata::get(ConstantInt::get(
+ Ty, AssumedConstantRange.getLower())),
+ ConstantAsMetadata::get(ConstantInt::get(
+ Ty, AssumedConstantRange.getUpper()))};
+ return MDNode::get(Ctx, LowAndHigh);
+ }
+
+ /// Return true if \p Assumed is included in \p KnownRanges.
+ static bool isBetterRange(const ConstantRange &Assumed, MDNode *KnownRanges) {
+
+ if (Assumed.isFullSet())
+ return false;
+
+ if (!KnownRanges)
+ return true;
+
+ // If multiple ranges are annotated in IR, we give up to annotate assumed
+ // range for now.
+
+ // TODO: If there exists a known range which containts assumed range, we
+ // can say assumed range is better.
+ if (KnownRanges->getNumOperands() > 2)
+ return false;
+
+ ConstantInt *Lower =
+ mdconst::extract<ConstantInt>(KnownRanges->getOperand(0));
+ ConstantInt *Upper =
+ mdconst::extract<ConstantInt>(KnownRanges->getOperand(1));
+
+ ConstantRange Known(Lower->getValue(), Upper->getValue());
+ return Known.contains(Assumed) && Known != Assumed;
+ }
+
+ /// Helper function to set range metadata.
+ static bool
+ setRangeMetadataIfisBetterRange(Instruction *I,
+ const ConstantRange &AssumedConstantRange) {
+ auto *OldRangeMD = I->getMetadata(LLVMContext::MD_range);
+ if (isBetterRange(AssumedConstantRange, OldRangeMD)) {
+ if (!AssumedConstantRange.isEmptySet()) {
+ I->setMetadata(LLVMContext::MD_range,
+ getMDNodeForConstantRange(I->getType(), I->getContext(),
+ AssumedConstantRange));
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /// See AbstractAttribute::manifest()
+ ChangeStatus manifest(Attributor &A) override {
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ ConstantRange AssumedConstantRange = getAssumedConstantRange(A);
+ assert(!AssumedConstantRange.isFullSet() && "Invalid state");
+
+ auto &V = getAssociatedValue();
+ if (!AssumedConstantRange.isEmptySet() &&
+ !AssumedConstantRange.isSingleElement()) {
+ if (Instruction *I = dyn_cast<Instruction>(&V)) {
+ assert(I == getCtxI() && "Should not annotate an instruction which is "
+ "not the context instruction");
+ if (isa<CallInst>(I) || isa<LoadInst>(I))
+ if (setRangeMetadataIfisBetterRange(I, AssumedConstantRange))
+ Changed = ChangeStatus::CHANGED;
+ }
+ }
+
+ return Changed;
+ }
+};
+
+struct AAValueConstantRangeArgument final
+ : AAArgumentFromCallSiteArguments<
+ AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
+ true /* BridgeCallBaseContext */> {
+ using Base = AAArgumentFromCallSiteArguments<
+ AAValueConstantRange, AAValueConstantRangeImpl, IntegerRangeState,
+ true /* BridgeCallBaseContext */>;
+ AAValueConstantRangeArgument(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_ARG_ATTR(value_range)
+ }
+};
+
+struct AAValueConstantRangeReturned
+ : AAReturnedFromReturnedValues<AAValueConstantRange,
+ AAValueConstantRangeImpl,
+ AAValueConstantRangeImpl::StateType,
+ /* PropogateCallBaseContext */ true> {
+ using Base =
+ AAReturnedFromReturnedValues<AAValueConstantRange,
+ AAValueConstantRangeImpl,
+ AAValueConstantRangeImpl::StateType,
+ /* PropogateCallBaseContext */ true>;
+ AAValueConstantRangeReturned(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))
+ indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FNRET_ATTR(value_range)
+ }
+};
+
+struct AAValueConstantRangeFloating : AAValueConstantRangeImpl {
+ AAValueConstantRangeFloating(const IRPosition &IRP, Attributor &A)
+ : AAValueConstantRangeImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ AAValueConstantRangeImpl::initialize(A);
+ if (isAtFixpoint())
+ return;
+
+ Value &V = getAssociatedValue();
+
+ if (auto *C = dyn_cast<ConstantInt>(&V)) {
+ unionAssumed(ConstantRange(C->getValue()));
+ indicateOptimisticFixpoint();
+ return;
+ }
+
+ if (isa<UndefValue>(&V)) {
+ // Collapse the undef state to 0.
+ unionAssumed(ConstantRange(APInt(getBitWidth(), 0)));
+ indicateOptimisticFixpoint();
+ return;
+ }
+
+ if (isa<CallBase>(&V))
+ return;
+
+ if (isa<BinaryOperator>(&V) || isa<CmpInst>(&V) || isa<CastInst>(&V))
+ return;
+
+ // If it is a load instruction with range metadata, use it.
+ if (LoadInst *LI = dyn_cast<LoadInst>(&V))
+ if (auto *RangeMD = LI->getMetadata(LLVMContext::MD_range)) {
+ intersectKnown(getConstantRangeFromMetadata(*RangeMD));
+ return;
+ }
+
+ // We can work with PHI and select instruction as we traverse their operands
+ // during update.
+ if (isa<SelectInst>(V) || isa<PHINode>(V))
+ return;
+
+ // Otherwise we give up.
+ indicatePessimisticFixpoint();
+
+ LLVM_DEBUG(dbgs() << "[AAValueConstantRange] We give up: "
+ << getAssociatedValue() << "\n");
+ }
+
+ bool calculateBinaryOperator(
+ Attributor &A, BinaryOperator *BinOp, IntegerRangeState &T,
+ const Instruction *CtxI,
+ SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
+ Value *LHS = BinOp->getOperand(0);
+ Value *RHS = BinOp->getOperand(1);
+
+ // Simplify the operands first.
+ bool UsedAssumedInformation = false;
+ const auto &SimplifiedLHS = A.getAssumedSimplified(
+ IRPosition::value(*LHS, getCallBaseContext()), *this,
+ UsedAssumedInformation, AA::Interprocedural);
+ if (!SimplifiedLHS.has_value())
+ return true;
+ if (!*SimplifiedLHS)
+ return false;
+ LHS = *SimplifiedLHS;
+
+ const auto &SimplifiedRHS = A.getAssumedSimplified(
+ IRPosition::value(*RHS, getCallBaseContext()), *this,
+ UsedAssumedInformation, AA::Interprocedural);
+ if (!SimplifiedRHS.has_value())
+ return true;
+ if (!*SimplifiedRHS)
+ return false;
+ RHS = *SimplifiedRHS;
+
+ // TODO: Allow non integers as well.
+ if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
+ return false;
+
+ auto *LHSAA = A.getAAFor<AAValueConstantRange>(
+ *this, IRPosition::value(*LHS, getCallBaseContext()),
+ DepClassTy::REQUIRED);
+ if (!LHSAA)
+ return false;
+ QuerriedAAs.push_back(LHSAA);
+ auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
+
+ auto *RHSAA = A.getAAFor<AAValueConstantRange>(
+ *this, IRPosition::value(*RHS, getCallBaseContext()),
+ DepClassTy::REQUIRED);
+ if (!RHSAA)
+ return false;
+ QuerriedAAs.push_back(RHSAA);
+ auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
+
+ auto AssumedRange = LHSAARange.binaryOp(BinOp->getOpcode(), RHSAARange);
+
+ T.unionAssumed(AssumedRange);
+
+ // TODO: Track a known state too.
+
+ return T.isValidState();
+ }
+
+ bool calculateCastInst(
+ Attributor &A, CastInst *CastI, IntegerRangeState &T,
+ const Instruction *CtxI,
+ SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
+ assert(CastI->getNumOperands() == 1 && "Expected cast to be unary!");
+ // TODO: Allow non integers as well.
+ Value *OpV = CastI->getOperand(0);
+
+ // Simplify the operand first.
+ bool UsedAssumedInformation = false;
+ const auto &SimplifiedOpV = A.getAssumedSimplified(
+ IRPosition::value(*OpV, getCallBaseContext()), *this,
+ UsedAssumedInformation, AA::Interprocedural);
+ if (!SimplifiedOpV.has_value())
+ return true;
+ if (!*SimplifiedOpV)
+ return false;
+ OpV = *SimplifiedOpV;
+
+ if (!OpV->getType()->isIntegerTy())
+ return false;
+
+ auto *OpAA = A.getAAFor<AAValueConstantRange>(
+ *this, IRPosition::value(*OpV, getCallBaseContext()),
+ DepClassTy::REQUIRED);
+ if (!OpAA)
+ return false;
+ QuerriedAAs.push_back(OpAA);
+ T.unionAssumed(OpAA->getAssumed().castOp(CastI->getOpcode(),
+ getState().getBitWidth()));
+ return T.isValidState();
+ }
+
+ bool
+ calculateCmpInst(Attributor &A, CmpInst *CmpI, IntegerRangeState &T,
+ const Instruction *CtxI,
+ SmallVectorImpl<const AAValueConstantRange *> &QuerriedAAs) {
+ Value *LHS = CmpI->getOperand(0);
+ Value *RHS = CmpI->getOperand(1);
+
+ // Simplify the operands first.
+ bool UsedAssumedInformation = false;
+ const auto &SimplifiedLHS = A.getAssumedSimplified(
+ IRPosition::value(*LHS, getCallBaseContext()), *this,
+ UsedAssumedInformation, AA::Interprocedural);
+ if (!SimplifiedLHS.has_value())
+ return true;
+ if (!*SimplifiedLHS)
+ return false;
+ LHS = *SimplifiedLHS;
+
+ const auto &SimplifiedRHS = A.getAssumedSimplified(
+ IRPosition::value(*RHS, getCallBaseContext()), *this,
+ UsedAssumedInformation, AA::Interprocedural);
+ if (!SimplifiedRHS.has_value())
+ return true;
+ if (!*SimplifiedRHS)
+ return false;
+ RHS = *SimplifiedRHS;
+
+ // TODO: Allow non integers as well.
+ if (!LHS->getType()->isIntegerTy() || !RHS->getType()->isIntegerTy())
+ return false;
+
+ auto *LHSAA = A.getAAFor<AAValueConstantRange>(
+ *this, IRPosition::value(*LHS, getCallBaseContext()),
+ DepClassTy::REQUIRED);
+ if (!LHSAA)
+ return false;
+ QuerriedAAs.push_back(LHSAA);
+ auto *RHSAA = A.getAAFor<AAValueConstantRange>(
+ *this, IRPosition::value(*RHS, getCallBaseContext()),
+ DepClassTy::REQUIRED);
+ if (!RHSAA)
+ return false;
+ QuerriedAAs.push_back(RHSAA);
+ auto LHSAARange = LHSAA->getAssumedConstantRange(A, CtxI);
+ auto RHSAARange = RHSAA->getAssumedConstantRange(A, CtxI);
+
+ // If one of them is empty set, we can't decide.
+ if (LHSAARange.isEmptySet() || RHSAARange.isEmptySet())
+ return true;
+
+ bool MustTrue = false, MustFalse = false;
+
+ auto AllowedRegion =
+ ConstantRange::makeAllowedICmpRegion(CmpI->getPredicate(), RHSAARange);
+
+ if (AllowedRegion.intersectWith(LHSAARange).isEmptySet())
+ MustFalse = true;
+
+ if (LHSAARange.icmp(CmpI->getPredicate(), RHSAARange))
+ MustTrue = true;
+
+ assert((!MustTrue || !MustFalse) &&
+ "Either MustTrue or MustFalse should be false!");
+
+ if (MustTrue)
+ T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 1)));
+ else if (MustFalse)
+ T.unionAssumed(ConstantRange(APInt(/* numBits */ 1, /* val */ 0)));
+ else
+ T.unionAssumed(ConstantRange(/* BitWidth */ 1, /* isFullSet */ true));
+
+ LLVM_DEBUG(dbgs() << "[AAValueConstantRange] " << *CmpI << " after "
+ << (MustTrue ? "true" : (MustFalse ? "false" : "unknown"))
+ << ": " << T << "\n\t" << *LHSAA << "\t<op>\n\t"
+ << *RHSAA);
+
+ // TODO: Track a known state too.
+ return T.isValidState();
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+
+ IntegerRangeState T(getBitWidth());
+ auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
+ Instruction *I = dyn_cast<Instruction>(&V);
+ if (!I || isa<CallBase>(I)) {
+
+ // Simplify the operand first.
+ bool UsedAssumedInformation = false;
+ const auto &SimplifiedOpV = A.getAssumedSimplified(
+ IRPosition::value(V, getCallBaseContext()), *this,
+ UsedAssumedInformation, AA::Interprocedural);
+ if (!SimplifiedOpV.has_value())
+ return true;
+ if (!*SimplifiedOpV)
+ return false;
+ Value *VPtr = *SimplifiedOpV;
+
+ // If the value is not instruction, we query AA to Attributor.
+ const auto *AA = A.getAAFor<AAValueConstantRange>(
+ *this, IRPosition::value(*VPtr, getCallBaseContext()),
+ DepClassTy::REQUIRED);
+
+ // Clamp operator is not used to utilize a program point CtxI.
+ if (AA)
+ T.unionAssumed(AA->getAssumedConstantRange(A, CtxI));
+ else
+ return false;
+
+ return T.isValidState();
+ }
+
+ SmallVector<const AAValueConstantRange *, 4> QuerriedAAs;
+ if (auto *BinOp = dyn_cast<BinaryOperator>(I)) {
+ if (!calculateBinaryOperator(A, BinOp, T, CtxI, QuerriedAAs))
+ return false;
+ } else if (auto *CmpI = dyn_cast<CmpInst>(I)) {
+ if (!calculateCmpInst(A, CmpI, T, CtxI, QuerriedAAs))
+ return false;
+ } else if (auto *CastI = dyn_cast<CastInst>(I)) {
+ if (!calculateCastInst(A, CastI, T, CtxI, QuerriedAAs))
+ return false;
+ } else {
+ // Give up with other instructions.
+ // TODO: Add other instructions
+
+ T.indicatePessimisticFixpoint();
+ return false;
+ }
+
+ // Catch circular reasoning in a pessimistic way for now.
+ // TODO: Check how the range evolves and if we stripped anything, see also
+ // AADereferenceable or AAAlign for similar situations.
+ for (const AAValueConstantRange *QueriedAA : QuerriedAAs) {
+ if (QueriedAA != this)
+ continue;
+ // If we are in a stady state we do not need to worry.
+ if (T.getAssumed() == getState().getAssumed())
+ continue;
+ T.indicatePessimisticFixpoint();
+ }
+
+ return T.isValidState();
+ };
+
+ if (!VisitValueCB(getAssociatedValue(), getCtxI()))
+ return indicatePessimisticFixpoint();
+
+ // Ensure that long def-use chains can't cause circular reasoning either by
+ // introducing a cutoff below.
+ if (clampStateAndIndicateChange(getState(), T) == ChangeStatus::UNCHANGED)
+ return ChangeStatus::UNCHANGED;
+ if (++NumChanges > MaxNumChanges) {
+ LLVM_DEBUG(dbgs() << "[AAValueConstantRange] performed " << NumChanges
+ << " but only " << MaxNumChanges
+ << " are allowed to avoid cyclic reasoning.");
+ return indicatePessimisticFixpoint();
+ }
+ return ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(value_range)
+ }
+
+ /// Tracker to bail after too many widening steps of the constant range.
+ int NumChanges = 0;
+
+ /// Upper bound for the number of allowed changes (=widening steps) for the
+ /// constant range before we give up.
+ static constexpr int MaxNumChanges = 5;
+};
+
+struct AAValueConstantRangeFunction : AAValueConstantRangeImpl {
+ AAValueConstantRangeFunction(const IRPosition &IRP, Attributor &A)
+ : AAValueConstantRangeImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ llvm_unreachable("AAValueConstantRange(Function|CallSite)::updateImpl will "
+ "not be called");
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(value_range) }
+};
+
+struct AAValueConstantRangeCallSite : AAValueConstantRangeFunction {
+ AAValueConstantRangeCallSite(const IRPosition &IRP, Attributor &A)
+ : AAValueConstantRangeFunction(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CS_ATTR(value_range) }
+};
+
+struct AAValueConstantRangeCallSiteReturned
+ : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
+ AAValueConstantRangeImpl::StateType,
+ /* IntroduceCallBaseContext */ true> {
+ AAValueConstantRangeCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AAValueConstantRange, AAValueConstantRangeImpl,
+ AAValueConstantRangeImpl::StateType,
+ /* IntroduceCallBaseContext */ true>(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ // If it is a load instruction with range metadata, use the metadata.
+ if (CallInst *CI = dyn_cast<CallInst>(&getAssociatedValue()))
+ if (auto *RangeMD = CI->getMetadata(LLVMContext::MD_range))
+ intersectKnown(getConstantRangeFromMetadata(*RangeMD));
+
+ AAValueConstantRangeImpl::initialize(A);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSRET_ATTR(value_range)
+ }
+};
+struct AAValueConstantRangeCallSiteArgument : AAValueConstantRangeFloating {
+ AAValueConstantRangeCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAValueConstantRangeFloating(IRP, A) {}
+
+ /// See AbstractAttribute::manifest()
+ ChangeStatus manifest(Attributor &A) override {
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSARG_ATTR(value_range)
+ }
+};
+} // namespace
+
+/// ------------------ Potential Values Attribute -------------------------
+
+namespace {
+struct AAPotentialConstantValuesImpl : AAPotentialConstantValues {
+ using StateType = PotentialConstantIntValuesState;
+
+ AAPotentialConstantValuesImpl(const IRPosition &IRP, Attributor &A)
+ : AAPotentialConstantValues(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(..).
+ void initialize(Attributor &A) override {
+ if (A.hasSimplificationCallback(getIRPosition()))
+ indicatePessimisticFixpoint();
+ else
+ AAPotentialConstantValues::initialize(A);
+ }
+
+ bool fillSetWithConstantValues(Attributor &A, const IRPosition &IRP, SetTy &S,
+ bool &ContainsUndef, bool ForSelf) {
+ SmallVector<AA::ValueAndContext> Values;
+ bool UsedAssumedInformation = false;
+ if (!A.getAssumedSimplifiedValues(IRP, *this, Values, AA::Interprocedural,
+ UsedAssumedInformation)) {
+ // Avoid recursion when the caller is computing constant values for this
+ // IRP itself.
+ if (ForSelf)
+ return false;
+ if (!IRP.getAssociatedType()->isIntegerTy())
+ return false;
+ auto *PotentialValuesAA = A.getAAFor<AAPotentialConstantValues>(
+ *this, IRP, DepClassTy::REQUIRED);
+ if (!PotentialValuesAA || !PotentialValuesAA->getState().isValidState())
+ return false;
+ ContainsUndef = PotentialValuesAA->getState().undefIsContained();
+ S = PotentialValuesAA->getState().getAssumedSet();
+ return true;
+ }
+
+ // Copy all the constant values, except UndefValue. ContainsUndef is true
+ // iff Values contains only UndefValue instances. If there are other known
+ // constants, then UndefValue is dropped.
+ ContainsUndef = false;
+ for (auto &It : Values) {
+ if (isa<UndefValue>(It.getValue())) {
+ ContainsUndef = true;
+ continue;
+ }
+ auto *CI = dyn_cast<ConstantInt>(It.getValue());
+ if (!CI)
+ return false;
+ S.insert(CI->getValue());
+ }
+ ContainsUndef &= S.empty();
+
+ return true;
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ std::string Str;
+ llvm::raw_string_ostream OS(Str);
+ OS << getState();
+ return OS.str();
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ return indicatePessimisticFixpoint();
+ }
+};
+
+struct AAPotentialConstantValuesArgument final
+ : AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
+ AAPotentialConstantValuesImpl,
+ PotentialConstantIntValuesState> {
+ using Base = AAArgumentFromCallSiteArguments<AAPotentialConstantValues,
+ AAPotentialConstantValuesImpl,
+ PotentialConstantIntValuesState>;
+ AAPotentialConstantValuesArgument(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_ARG_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialConstantValuesReturned
+ : AAReturnedFromReturnedValues<AAPotentialConstantValues,
+ AAPotentialConstantValuesImpl> {
+ using Base = AAReturnedFromReturnedValues<AAPotentialConstantValues,
+ AAPotentialConstantValuesImpl>;
+ AAPotentialConstantValuesReturned(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ void initialize(Attributor &A) override {
+ if (!A.isFunctionIPOAmendable(*getAssociatedFunction()))
+ indicatePessimisticFixpoint();
+ Base::initialize(A);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FNRET_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialConstantValuesFloating : AAPotentialConstantValuesImpl {
+ AAPotentialConstantValuesFloating(const IRPosition &IRP, Attributor &A)
+ : AAPotentialConstantValuesImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(..).
+ void initialize(Attributor &A) override {
+ AAPotentialConstantValuesImpl::initialize(A);
+ if (isAtFixpoint())
+ return;
+
+ Value &V = getAssociatedValue();
+
+ if (auto *C = dyn_cast<ConstantInt>(&V)) {
+ unionAssumed(C->getValue());
+ indicateOptimisticFixpoint();
+ return;
+ }
+
+ if (isa<UndefValue>(&V)) {
+ unionAssumedWithUndef();
+ indicateOptimisticFixpoint();
+ return;
+ }
+
+ if (isa<BinaryOperator>(&V) || isa<ICmpInst>(&V) || isa<CastInst>(&V))
+ return;
+
+ if (isa<SelectInst>(V) || isa<PHINode>(V) || isa<LoadInst>(V))
+ return;
+
+ indicatePessimisticFixpoint();
+
+ LLVM_DEBUG(dbgs() << "[AAPotentialConstantValues] We give up: "
+ << getAssociatedValue() << "\n");
+ }
+
+ static bool calculateICmpInst(const ICmpInst *ICI, const APInt &LHS,
+ const APInt &RHS) {
+ return ICmpInst::compare(LHS, RHS, ICI->getPredicate());
+ }
+
+ static APInt calculateCastInst(const CastInst *CI, const APInt &Src,
+ uint32_t ResultBitWidth) {
+ Instruction::CastOps CastOp = CI->getOpcode();
+ switch (CastOp) {
+ default:
+ llvm_unreachable("unsupported or not integer cast");
+ case Instruction::Trunc:
+ return Src.trunc(ResultBitWidth);
+ case Instruction::SExt:
+ return Src.sext(ResultBitWidth);
+ case Instruction::ZExt:
+ return Src.zext(ResultBitWidth);
+ case Instruction::BitCast:
+ return Src;
+ }
+ }
+
+ static APInt calculateBinaryOperator(const BinaryOperator *BinOp,
+ const APInt &LHS, const APInt &RHS,
+ bool &SkipOperation, bool &Unsupported) {
+ Instruction::BinaryOps BinOpcode = BinOp->getOpcode();
+ // Unsupported is set to true when the binary operator is not supported.
+ // SkipOperation is set to true when UB occur with the given operand pair
+ // (LHS, RHS).
+ // TODO: we should look at nsw and nuw keywords to handle operations
+ // that create poison or undef value.
+ switch (BinOpcode) {
+ default:
+ Unsupported = true;
+ return LHS;
+ case Instruction::Add:
+ return LHS + RHS;
+ case Instruction::Sub:
+ return LHS - RHS;
+ case Instruction::Mul:
+ return LHS * RHS;
+ case Instruction::UDiv:
+ if (RHS.isZero()) {
+ SkipOperation = true;
+ return LHS;
+ }
+ return LHS.udiv(RHS);
+ case Instruction::SDiv:
+ if (RHS.isZero()) {
+ SkipOperation = true;
+ return LHS;
+ }
+ return LHS.sdiv(RHS);
+ case Instruction::URem:
+ if (RHS.isZero()) {
+ SkipOperation = true;
+ return LHS;
+ }
+ return LHS.urem(RHS);
+ case Instruction::SRem:
+ if (RHS.isZero()) {
+ SkipOperation = true;
+ return LHS;
+ }
+ return LHS.srem(RHS);
+ case Instruction::Shl:
+ return LHS.shl(RHS);
+ case Instruction::LShr:
+ return LHS.lshr(RHS);
+ case Instruction::AShr:
+ return LHS.ashr(RHS);
+ case Instruction::And:
+ return LHS & RHS;
+ case Instruction::Or:
+ return LHS | RHS;
+ case Instruction::Xor:
+ return LHS ^ RHS;
+ }
+ }
+
+ bool calculateBinaryOperatorAndTakeUnion(const BinaryOperator *BinOp,
+ const APInt &LHS, const APInt &RHS) {
+ bool SkipOperation = false;
+ bool Unsupported = false;
+ APInt Result =
+ calculateBinaryOperator(BinOp, LHS, RHS, SkipOperation, Unsupported);
+ if (Unsupported)
+ return false;
+ // If SkipOperation is true, we can ignore this operand pair (L, R).
+ if (!SkipOperation)
+ unionAssumed(Result);
+ return isValidState();
+ }
+
+ ChangeStatus updateWithICmpInst(Attributor &A, ICmpInst *ICI) {
+ auto AssumedBefore = getAssumed();
+ Value *LHS = ICI->getOperand(0);
+ Value *RHS = ICI->getOperand(1);
+
+ bool LHSContainsUndef = false, RHSContainsUndef = false;
+ SetTy LHSAAPVS, RHSAAPVS;
+ if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
+ LHSContainsUndef, /* ForSelf */ false) ||
+ !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
+ RHSContainsUndef, /* ForSelf */ false))
+ return indicatePessimisticFixpoint();
+
+ // TODO: make use of undef flag to limit potential values aggressively.
+ bool MaybeTrue = false, MaybeFalse = false;
+ const APInt Zero(RHS->getType()->getIntegerBitWidth(), 0);
+ if (LHSContainsUndef && RHSContainsUndef) {
+ // The result of any comparison between undefs can be soundly replaced
+ // with undef.
+ unionAssumedWithUndef();
+ } else if (LHSContainsUndef) {
+ for (const APInt &R : RHSAAPVS) {
+ bool CmpResult = calculateICmpInst(ICI, Zero, R);
+ MaybeTrue |= CmpResult;
+ MaybeFalse |= !CmpResult;
+ if (MaybeTrue & MaybeFalse)
+ return indicatePessimisticFixpoint();
+ }
+ } else if (RHSContainsUndef) {
+ for (const APInt &L : LHSAAPVS) {
+ bool CmpResult = calculateICmpInst(ICI, L, Zero);
+ MaybeTrue |= CmpResult;
+ MaybeFalse |= !CmpResult;
+ if (MaybeTrue & MaybeFalse)
+ return indicatePessimisticFixpoint();
+ }
+ } else {
+ for (const APInt &L : LHSAAPVS) {
+ for (const APInt &R : RHSAAPVS) {
+ bool CmpResult = calculateICmpInst(ICI, L, R);
+ MaybeTrue |= CmpResult;
+ MaybeFalse |= !CmpResult;
+ if (MaybeTrue & MaybeFalse)
+ return indicatePessimisticFixpoint();
+ }
+ }
+ }
+ if (MaybeTrue)
+ unionAssumed(APInt(/* numBits */ 1, /* val */ 1));
+ if (MaybeFalse)
+ unionAssumed(APInt(/* numBits */ 1, /* val */ 0));
+ return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ ChangeStatus updateWithSelectInst(Attributor &A, SelectInst *SI) {
+ auto AssumedBefore = getAssumed();
+ Value *LHS = SI->getTrueValue();
+ Value *RHS = SI->getFalseValue();
+
+ bool UsedAssumedInformation = false;
+ std::optional<Constant *> C = A.getAssumedConstant(
+ *SI->getCondition(), *this, UsedAssumedInformation);
+
+ // Check if we only need one operand.
+ bool OnlyLeft = false, OnlyRight = false;
+ if (C && *C && (*C)->isOneValue())
+ OnlyLeft = true;
+ else if (C && *C && (*C)->isZeroValue())
+ OnlyRight = true;
+
+ bool LHSContainsUndef = false, RHSContainsUndef = false;
+ SetTy LHSAAPVS, RHSAAPVS;
+ if (!OnlyRight &&
+ !fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
+ LHSContainsUndef, /* ForSelf */ false))
+ return indicatePessimisticFixpoint();
+
+ if (!OnlyLeft &&
+ !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
+ RHSContainsUndef, /* ForSelf */ false))
+ return indicatePessimisticFixpoint();
+
+ if (OnlyLeft || OnlyRight) {
+ // select (true/false), lhs, rhs
+ auto *OpAA = OnlyLeft ? &LHSAAPVS : &RHSAAPVS;
+ auto Undef = OnlyLeft ? LHSContainsUndef : RHSContainsUndef;
+
+ if (Undef)
+ unionAssumedWithUndef();
+ else {
+ for (const auto &It : *OpAA)
+ unionAssumed(It);
+ }
+
+ } else if (LHSContainsUndef && RHSContainsUndef) {
+ // select i1 *, undef , undef => undef
+ unionAssumedWithUndef();
+ } else {
+ for (const auto &It : LHSAAPVS)
+ unionAssumed(It);
+ for (const auto &It : RHSAAPVS)
+ unionAssumed(It);
+ }
+ return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ ChangeStatus updateWithCastInst(Attributor &A, CastInst *CI) {
+ auto AssumedBefore = getAssumed();
+ if (!CI->isIntegerCast())
+ return indicatePessimisticFixpoint();
+ assert(CI->getNumOperands() == 1 && "Expected cast to be unary!");
+ uint32_t ResultBitWidth = CI->getDestTy()->getIntegerBitWidth();
+ Value *Src = CI->getOperand(0);
+
+ bool SrcContainsUndef = false;
+ SetTy SrcPVS;
+ if (!fillSetWithConstantValues(A, IRPosition::value(*Src), SrcPVS,
+ SrcContainsUndef, /* ForSelf */ false))
+ return indicatePessimisticFixpoint();
+
+ if (SrcContainsUndef)
+ unionAssumedWithUndef();
+ else {
+ for (const APInt &S : SrcPVS) {
+ APInt T = calculateCastInst(CI, S, ResultBitWidth);
+ unionAssumed(T);
+ }
+ }
+ return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ ChangeStatus updateWithBinaryOperator(Attributor &A, BinaryOperator *BinOp) {
+ auto AssumedBefore = getAssumed();
+ Value *LHS = BinOp->getOperand(0);
+ Value *RHS = BinOp->getOperand(1);
+
+ bool LHSContainsUndef = false, RHSContainsUndef = false;
+ SetTy LHSAAPVS, RHSAAPVS;
+ if (!fillSetWithConstantValues(A, IRPosition::value(*LHS), LHSAAPVS,
+ LHSContainsUndef, /* ForSelf */ false) ||
+ !fillSetWithConstantValues(A, IRPosition::value(*RHS), RHSAAPVS,
+ RHSContainsUndef, /* ForSelf */ false))
+ return indicatePessimisticFixpoint();
+
+ const APInt Zero = APInt(LHS->getType()->getIntegerBitWidth(), 0);
+
+ // TODO: make use of undef flag to limit potential values aggressively.
+ if (LHSContainsUndef && RHSContainsUndef) {
+ if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, Zero))
+ return indicatePessimisticFixpoint();
+ } else if (LHSContainsUndef) {
+ for (const APInt &R : RHSAAPVS) {
+ if (!calculateBinaryOperatorAndTakeUnion(BinOp, Zero, R))
+ return indicatePessimisticFixpoint();
+ }
+ } else if (RHSContainsUndef) {
+ for (const APInt &L : LHSAAPVS) {
+ if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, Zero))
+ return indicatePessimisticFixpoint();
+ }
+ } else {
+ for (const APInt &L : LHSAAPVS) {
+ for (const APInt &R : RHSAAPVS) {
+ if (!calculateBinaryOperatorAndTakeUnion(BinOp, L, R))
+ return indicatePessimisticFixpoint();
+ }
+ }
+ }
+ return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ ChangeStatus updateWithInstruction(Attributor &A, Instruction *Inst) {
+ auto AssumedBefore = getAssumed();
+ SetTy Incoming;
+ bool ContainsUndef;
+ if (!fillSetWithConstantValues(A, IRPosition::value(*Inst), Incoming,
+ ContainsUndef, /* ForSelf */ true))
+ return indicatePessimisticFixpoint();
+ if (ContainsUndef) {
+ unionAssumedWithUndef();
+ } else {
+ for (const auto &It : Incoming)
+ unionAssumed(It);
+ }
+ return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ Value &V = getAssociatedValue();
+ Instruction *I = dyn_cast<Instruction>(&V);
+
+ if (auto *ICI = dyn_cast<ICmpInst>(I))
+ return updateWithICmpInst(A, ICI);
+
+ if (auto *SI = dyn_cast<SelectInst>(I))
+ return updateWithSelectInst(A, SI);
+
+ if (auto *CI = dyn_cast<CastInst>(I))
+ return updateWithCastInst(A, CI);
+
+ if (auto *BinOp = dyn_cast<BinaryOperator>(I))
+ return updateWithBinaryOperator(A, BinOp);
+
+ if (isa<PHINode>(I) || isa<LoadInst>(I))
+ return updateWithInstruction(A, I);
+
+ return indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialConstantValuesFunction : AAPotentialConstantValuesImpl {
+ AAPotentialConstantValuesFunction(const IRPosition &IRP, Attributor &A)
+ : AAPotentialConstantValuesImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ llvm_unreachable(
+ "AAPotentialConstantValues(Function|CallSite)::updateImpl will "
+ "not be called");
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FN_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialConstantValuesCallSite : AAPotentialConstantValuesFunction {
+ AAPotentialConstantValuesCallSite(const IRPosition &IRP, Attributor &A)
+ : AAPotentialConstantValuesFunction(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CS_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialConstantValuesCallSiteReturned
+ : AACalleeToCallSite<AAPotentialConstantValues,
+ AAPotentialConstantValuesImpl> {
+ AAPotentialConstantValuesCallSiteReturned(const IRPosition &IRP,
+ Attributor &A)
+ : AACalleeToCallSite<AAPotentialConstantValues,
+ AAPotentialConstantValuesImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSRET_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialConstantValuesCallSiteArgument
+ : AAPotentialConstantValuesFloating {
+ AAPotentialConstantValuesCallSiteArgument(const IRPosition &IRP,
+ Attributor &A)
+ : AAPotentialConstantValuesFloating(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(..).
+ void initialize(Attributor &A) override {
+ AAPotentialConstantValuesImpl::initialize(A);
+ if (isAtFixpoint())
+ return;
+
+ Value &V = getAssociatedValue();
+
+ if (auto *C = dyn_cast<ConstantInt>(&V)) {
+ unionAssumed(C->getValue());
+ indicateOptimisticFixpoint();
+ return;
+ }
+
+ if (isa<UndefValue>(&V)) {
+ unionAssumedWithUndef();
+ indicateOptimisticFixpoint();
+ return;
+ }
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ Value &V = getAssociatedValue();
+ auto AssumedBefore = getAssumed();
+ auto *AA = A.getAAFor<AAPotentialConstantValues>(
+ *this, IRPosition::value(V), DepClassTy::REQUIRED);
+ if (!AA)
+ return indicatePessimisticFixpoint();
+ const auto &S = AA->getAssumed();
+ unionAssumed(S);
+ return AssumedBefore == getAssumed() ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSARG_ATTR(potential_values)
+ }
+};
+} // namespace
+
+/// ------------------------ NoUndef Attribute ---------------------------------
+bool AANoUndef::isImpliedByIR(Attributor &A, const IRPosition &IRP,
+ Attribute::AttrKind ImpliedAttributeKind,
+ bool IgnoreSubsumingPositions) {
+ assert(ImpliedAttributeKind == Attribute::NoUndef &&
+ "Unexpected attribute kind");
+ if (A.hasAttr(IRP, {Attribute::NoUndef}, IgnoreSubsumingPositions,
+ Attribute::NoUndef))
+ return true;
+
+ Value &Val = IRP.getAssociatedValue();
+ if (IRP.getPositionKind() != IRPosition::IRP_RETURNED &&
+ isGuaranteedNotToBeUndefOrPoison(&Val)) {
+ LLVMContext &Ctx = Val.getContext();
+ A.manifestAttrs(IRP, Attribute::get(Ctx, Attribute::NoUndef));
+ return true;
+ }
+
+ return false;
+}
+
+namespace {
+struct AANoUndefImpl : AANoUndef {
+ AANoUndefImpl(const IRPosition &IRP, Attributor &A) : AANoUndef(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ Value &V = getAssociatedValue();
+ if (isa<UndefValue>(V))
+ indicatePessimisticFixpoint();
+ assert(!isImpliedByIR(A, getIRPosition(), Attribute::NoUndef));
+ }
+
+ /// See followUsesInMBEC
+ bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
+ AANoUndef::StateType &State) {
+ const Value *UseV = U->get();
+ const DominatorTree *DT = nullptr;
+ AssumptionCache *AC = nullptr;
+ InformationCache &InfoCache = A.getInfoCache();
+ if (Function *F = getAnchorScope()) {
+ DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);
+ AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
+ }
+ State.setKnown(isGuaranteedNotToBeUndefOrPoison(UseV, AC, I, DT));
+ bool TrackUse = false;
+ // Track use for instructions which must produce undef or poison bits when
+ // at least one operand contains such bits.
+ if (isa<CastInst>(*I) || isa<GetElementPtrInst>(*I))
+ TrackUse = true;
+ return TrackUse;
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return getAssumed() ? "noundef" : "may-undef-or-poison";
+ }
+
+ ChangeStatus manifest(Attributor &A) override {
+ // We don't manifest noundef attribute for dead positions because the
+ // associated values with dead positions would be replaced with undef
+ // values.
+ bool UsedAssumedInformation = false;
+ if (A.isAssumedDead(getIRPosition(), nullptr, nullptr,
+ UsedAssumedInformation))
+ return ChangeStatus::UNCHANGED;
+ // A position whose simplified value does not have any value is
+ // considered to be dead. We don't manifest noundef in such positions for
+ // the same reason above.
+ if (!A.getAssumedSimplified(getIRPosition(), *this, UsedAssumedInformation,
+ AA::Interprocedural)
+ .has_value())
+ return ChangeStatus::UNCHANGED;
+ return AANoUndef::manifest(A);
+ }
+};
+
+struct AANoUndefFloating : public AANoUndefImpl {
+ AANoUndefFloating(const IRPosition &IRP, Attributor &A)
+ : AANoUndefImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ AANoUndefImpl::initialize(A);
+ if (!getState().isAtFixpoint() && getAnchorScope() &&
+ !getAnchorScope()->isDeclaration())
+ if (Instruction *CtxI = getCtxI())
+ followUsesInMBEC(*this, A, getState(), *CtxI);
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto VisitValueCB = [&](const IRPosition &IRP) -> bool {
+ bool IsKnownNoUndef;
+ return AA::hasAssumedIRAttr<Attribute::NoUndef>(
+ A, this, IRP, DepClassTy::REQUIRED, IsKnownNoUndef);
+ };
+
+ bool Stripped;
+ bool UsedAssumedInformation = false;
+ Value *AssociatedValue = &getAssociatedValue();
+ SmallVector<AA::ValueAndContext> Values;
+ if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
+ AA::AnyScope, UsedAssumedInformation))
+ Stripped = false;
+ else
+ Stripped =
+ Values.size() != 1 || Values.front().getValue() != AssociatedValue;
+
+ if (!Stripped) {
+ // If we haven't stripped anything we might still be able to use a
+ // different AA, but only if the IRP changes. Effectively when we
+ // interpret this not as a call site value but as a floating/argument
+ // value.
+ const IRPosition AVIRP = IRPosition::value(*AssociatedValue);
+ if (AVIRP == getIRPosition() || !VisitValueCB(AVIRP))
+ return indicatePessimisticFixpoint();
+ return ChangeStatus::UNCHANGED;
+ }
+
+ for (const auto &VAC : Values)
+ if (!VisitValueCB(IRPosition::value(*VAC.getValue())))
+ return indicatePessimisticFixpoint();
+
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
+};
+
+struct AANoUndefReturned final
+ : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl> {
+ AANoUndefReturned(const IRPosition &IRP, Attributor &A)
+ : AAReturnedFromReturnedValues<AANoUndef, AANoUndefImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_FNRET_ATTR(noundef) }
+};
+
+struct AANoUndefArgument final
+ : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl> {
+ AANoUndefArgument(const IRPosition &IRP, Attributor &A)
+ : AAArgumentFromCallSiteArguments<AANoUndef, AANoUndefImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(noundef) }
+};
+
+struct AANoUndefCallSiteArgument final : AANoUndefFloating {
+ AANoUndefCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AANoUndefFloating(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CSARG_ATTR(noundef) }
+};
+
+struct AANoUndefCallSiteReturned final
+ : AACalleeToCallSite<AANoUndef, AANoUndefImpl> {
+ AANoUndefCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AANoUndef, AANoUndefImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_CSRET_ATTR(noundef) }
+};
+
+/// ------------------------ NoFPClass Attribute -------------------------------
+
+struct AANoFPClassImpl : AANoFPClass {
+ AANoFPClassImpl(const IRPosition &IRP, Attributor &A) : AANoFPClass(IRP, A) {}
+
+ void initialize(Attributor &A) override {
+ const IRPosition &IRP = getIRPosition();
+
+ Value &V = IRP.getAssociatedValue();
+ if (isa<UndefValue>(V)) {
+ indicateOptimisticFixpoint();
+ return;
+ }
+
+ SmallVector<Attribute> Attrs;
+ A.getAttrs(getIRPosition(), {Attribute::NoFPClass}, Attrs, false);
+ for (const auto &Attr : Attrs) {
+ addKnownBits(Attr.getNoFPClass());
+ }
+
+ const DataLayout &DL = A.getDataLayout();
+ if (getPositionKind() != IRPosition::IRP_RETURNED) {
+ KnownFPClass KnownFPClass = computeKnownFPClass(&V, DL);
+ addKnownBits(~KnownFPClass.KnownFPClasses);
+ }
+
+ if (Instruction *CtxI = getCtxI())
+ followUsesInMBEC(*this, A, getState(), *CtxI);
+ }
+
+ /// See followUsesInMBEC
+ bool followUseInMBEC(Attributor &A, const Use *U, const Instruction *I,
+ AANoFPClass::StateType &State) {
+ const Value *UseV = U->get();
+ const DominatorTree *DT = nullptr;
+ AssumptionCache *AC = nullptr;
+ const TargetLibraryInfo *TLI = nullptr;
+ InformationCache &InfoCache = A.getInfoCache();
+
+ if (Function *F = getAnchorScope()) {
+ DT = InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);
+ AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
+ TLI = InfoCache.getTargetLibraryInfoForFunction(*F);
+ }
+
+ const DataLayout &DL = A.getDataLayout();
+
+ KnownFPClass KnownFPClass =
+ computeKnownFPClass(UseV, DL,
+ /*InterestedClasses=*/fcAllFlags,
+ /*Depth=*/0, TLI, AC, I, DT);
+ State.addKnownBits(~KnownFPClass.KnownFPClasses);
+
+ if (auto *CI = dyn_cast<CallInst>(UseV)) {
+ // Special case FP intrinsic with struct return type.
+ switch (CI->getIntrinsicID()) {
+ case Intrinsic::frexp:
+ return true;
+ case Intrinsic::not_intrinsic:
+ // TODO: Could recognize math libcalls
+ return false;
+ default:
+ break;
+ }
+ }
+
+ if (!UseV->getType()->isFPOrFPVectorTy())
+ return false;
+ return !isa<LoadInst, AtomicRMWInst>(UseV);
+ }
+
+ const std::string getAsStr(Attributor *A) const override {
+ std::string Result = "nofpclass";
+ raw_string_ostream OS(Result);
+ OS << getAssumedNoFPClass();
+ return Result;
+ }
+
+ void getDeducedAttributes(Attributor &A, LLVMContext &Ctx,
+ SmallVectorImpl<Attribute> &Attrs) const override {
+ Attrs.emplace_back(Attribute::getWithNoFPClass(Ctx, getAssumedNoFPClass()));
+ }
+};
+
+struct AANoFPClassFloating : public AANoFPClassImpl {
+ AANoFPClassFloating(const IRPosition &IRP, Attributor &A)
+ : AANoFPClassImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ SmallVector<AA::ValueAndContext> Values;
+ bool UsedAssumedInformation = false;
+ if (!A.getAssumedSimplifiedValues(getIRPosition(), *this, Values,
+ AA::AnyScope, UsedAssumedInformation)) {
+ Values.push_back({getAssociatedValue(), getCtxI()});
+ }
+
+ StateType T;
+ auto VisitValueCB = [&](Value &V, const Instruction *CtxI) -> bool {
+ const auto *AA = A.getAAFor<AANoFPClass>(*this, IRPosition::value(V),
+ DepClassTy::REQUIRED);
+ if (!AA || this == AA) {
+ T.indicatePessimisticFixpoint();
+ } else {
+ const AANoFPClass::StateType &S =
+ static_cast<const AANoFPClass::StateType &>(AA->getState());
+ T ^= S;
+ }
+ return T.isValidState();
+ };
+
+ for (const auto &VAC : Values)
+ if (!VisitValueCB(*VAC.getValue(), VAC.getCtxI()))
+ return indicatePessimisticFixpoint();
+
+ return clampStateAndIndicateChange(getState(), T);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FNRET_ATTR(nofpclass)
+ }
+};
+
+struct AANoFPClassReturned final
+ : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
+ AANoFPClassImpl::StateType, false, Attribute::None, false> {
+ AANoFPClassReturned(const IRPosition &IRP, Attributor &A)
+ : AAReturnedFromReturnedValues<AANoFPClass, AANoFPClassImpl,
+ AANoFPClassImpl::StateType, false, Attribute::None, false>(
+ IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FNRET_ATTR(nofpclass)
+ }
+};
+
+struct AANoFPClassArgument final
+ : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl> {
+ AANoFPClassArgument(const IRPosition &IRP, Attributor &A)
+ : AAArgumentFromCallSiteArguments<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(nofpclass) }
+};
+
+struct AANoFPClassCallSiteArgument final : AANoFPClassFloating {
+ AANoFPClassCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AANoFPClassFloating(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSARG_ATTR(nofpclass)
+ }
+};
+
+struct AANoFPClassCallSiteReturned final
+ : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl> {
+ AANoFPClassCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AACalleeToCallSite<AANoFPClass, AANoFPClassImpl>(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSRET_ATTR(nofpclass)
+ }
+};
+
+struct AACallEdgesImpl : public AACallEdges {
+ AACallEdgesImpl(const IRPosition &IRP, Attributor &A) : AACallEdges(IRP, A) {}
+
+ const SetVector<Function *> &getOptimisticEdges() const override {
+ return CalledFunctions;
+ }
+
+ bool hasUnknownCallee() const override { return HasUnknownCallee; }
+
+ bool hasNonAsmUnknownCallee() const override {
+ return HasUnknownCalleeNonAsm;
+ }
+
+ const std::string getAsStr(Attributor *A) const override {
+ return "CallEdges[" + std::to_string(HasUnknownCallee) + "," +
+ std::to_string(CalledFunctions.size()) + "]";
+ }
+
+ void trackStatistics() const override {}
+
+protected:
+ void addCalledFunction(Function *Fn, ChangeStatus &Change) {
+ if (CalledFunctions.insert(Fn)) {
+ Change = ChangeStatus::CHANGED;
+ LLVM_DEBUG(dbgs() << "[AACallEdges] New call edge: " << Fn->getName()
+ << "\n");
+ }
+ }
+
+ void setHasUnknownCallee(bool NonAsm, ChangeStatus &Change) {
+ if (!HasUnknownCallee)
+ Change = ChangeStatus::CHANGED;
+ if (NonAsm && !HasUnknownCalleeNonAsm)
+ Change = ChangeStatus::CHANGED;
+ HasUnknownCalleeNonAsm |= NonAsm;
+ HasUnknownCallee = true;
+ }
+
+private:
+ /// Optimistic set of functions that might be called by this position.
+ SetVector<Function *> CalledFunctions;
+
+ /// Is there any call with a unknown callee.
+ bool HasUnknownCallee = false;
+
+ /// Is there any call with a unknown callee, excluding any inline asm.
+ bool HasUnknownCalleeNonAsm = false;
+};
+
+struct AACallEdgesCallSite : public AACallEdgesImpl {
+ AACallEdgesCallSite(const IRPosition &IRP, Attributor &A)
+ : AACallEdgesImpl(IRP, A) {}
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ ChangeStatus Change = ChangeStatus::UNCHANGED;
+
+ auto VisitValue = [&](Value &V, const Instruction *CtxI) -> bool {
+ if (Function *Fn = dyn_cast<Function>(&V)) {
+ addCalledFunction(Fn, Change);
+ } else {
+ LLVM_DEBUG(dbgs() << "[AACallEdges] Unrecognized value: " << V << "\n");
+ setHasUnknownCallee(true, Change);
+ }
+
+ // Explore all values.
+ return true;
+ };
+
+ SmallVector<AA::ValueAndContext> Values;
+ // Process any value that we might call.
+ auto ProcessCalledOperand = [&](Value *V, Instruction *CtxI) {
+ if (isa<Constant>(V)) {
+ VisitValue(*V, CtxI);
+ return;
+ }
+
+ bool UsedAssumedInformation = false;
+ Values.clear();
+ if (!A.getAssumedSimplifiedValues(IRPosition::value(*V), *this, Values,
+ AA::AnyScope, UsedAssumedInformation)) {
+ Values.push_back({*V, CtxI});
+ }
+ for (auto &VAC : Values)
+ VisitValue(*VAC.getValue(), VAC.getCtxI());
+ };
+
+ CallBase *CB = cast<CallBase>(getCtxI());
+
+ if (auto *IA = dyn_cast<InlineAsm>(CB->getCalledOperand())) {
+ if (IA->hasSideEffects() &&
+ !hasAssumption(*CB->getCaller(), "ompx_no_call_asm") &&
+ !hasAssumption(*CB, "ompx_no_call_asm")) {
+ setHasUnknownCallee(false, Change);
+ }
+ return Change;
+ }
+
+ if (CB->isIndirectCall())
+ if (auto *IndirectCallAA = A.getAAFor<AAIndirectCallInfo>(
+ *this, getIRPosition(), DepClassTy::OPTIONAL))
+ if (IndirectCallAA->foreachCallee(
+ [&](Function *Fn) { return VisitValue(*Fn, CB); }))
+ return Change;
+
+ // The most simple case.
+ ProcessCalledOperand(CB->getCalledOperand(), CB);
+
+ // Process callback functions.
+ SmallVector<const Use *, 4u> CallbackUses;
+ AbstractCallSite::getCallbackUses(*CB, CallbackUses);
+ for (const Use *U : CallbackUses)
+ ProcessCalledOperand(U->get(), CB);
+
+ return Change;
+ }
+};
+
+struct AACallEdgesFunction : public AACallEdgesImpl {
+ AACallEdgesFunction(const IRPosition &IRP, Attributor &A)
+ : AACallEdgesImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ ChangeStatus Change = ChangeStatus::UNCHANGED;
+
+ auto ProcessCallInst = [&](Instruction &Inst) {
+ CallBase &CB = cast<CallBase>(Inst);
+
+ auto *CBEdges = A.getAAFor<AACallEdges>(
+ *this, IRPosition::callsite_function(CB), DepClassTy::REQUIRED);
+ if (!CBEdges)
+ return false;
+ if (CBEdges->hasNonAsmUnknownCallee())
+ setHasUnknownCallee(true, Change);
+ if (CBEdges->hasUnknownCallee())
+ setHasUnknownCallee(false, Change);
+
+ for (Function *F : CBEdges->getOptimisticEdges())
+ addCalledFunction(F, Change);
+
+ return true;
+ };
+
+ // Visit all callable instructions.
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllCallLikeInstructions(ProcessCallInst, *this,
+ UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ true)) {
+ // If we haven't looked at all call like instructions, assume that there
+ // are unknown callees.
+ setHasUnknownCallee(true, Change);
+ }
+
+ return Change;
+ }
+};
+
+/// -------------------AAInterFnReachability Attribute--------------------------
+
+struct AAInterFnReachabilityFunction
+ : public CachedReachabilityAA<AAInterFnReachability, Function> {
+ using Base = CachedReachabilityAA<AAInterFnReachability, Function>;
+ AAInterFnReachabilityFunction(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ bool instructionCanReach(
+ Attributor &A, const Instruction &From, const Function &To,
+ const AA::InstExclusionSetTy *ExclusionSet) const override {
+ assert(From.getFunction() == getAnchorScope() && "Queried the wrong AA!");
+ auto *NonConstThis = const_cast<AAInterFnReachabilityFunction *>(this);
+
+ RQITy StackRQI(A, From, To, ExclusionSet, false);
+ typename RQITy::Reachable Result;
+ if (!NonConstThis->checkQueryCache(A, StackRQI, Result))
+ return NonConstThis->isReachableImpl(A, StackRQI,
+ /*IsTemporaryRQI=*/true);
+ return Result == RQITy::Reachable::Yes;
+ }
+
+ bool isReachableImpl(Attributor &A, RQITy &RQI,
+ bool IsTemporaryRQI) override {
+ const Instruction *EntryI =
+ &RQI.From->getFunction()->getEntryBlock().front();
+ if (EntryI != RQI.From &&
+ !instructionCanReach(A, *EntryI, *RQI.To, nullptr))
+ return rememberResult(A, RQITy::Reachable::No, RQI, false,
+ IsTemporaryRQI);
+
+ auto CheckReachableCallBase = [&](CallBase *CB) {
+ auto *CBEdges = A.getAAFor<AACallEdges>(
+ *this, IRPosition::callsite_function(*CB), DepClassTy::OPTIONAL);
+ if (!CBEdges || !CBEdges->getState().isValidState())
+ return false;
+ // TODO Check To backwards in this case.
+ if (CBEdges->hasUnknownCallee())
+ return false;
+
+ for (Function *Fn : CBEdges->getOptimisticEdges()) {
+ if (Fn == RQI.To)
+ return false;
+
+ if (Fn->isDeclaration()) {
+ if (Fn->hasFnAttribute(Attribute::NoCallback))
+ continue;
+ // TODO Check To backwards in this case.
+ return false;
+ }
+
+ if (Fn == getAnchorScope()) {
+ if (EntryI == RQI.From)
+ continue;
+ return false;
+ }
+
+ const AAInterFnReachability *InterFnReachability =
+ A.getAAFor<AAInterFnReachability>(*this, IRPosition::function(*Fn),
+ DepClassTy::OPTIONAL);
+
+ const Instruction &FnFirstInst = Fn->getEntryBlock().front();
+ if (!InterFnReachability ||
+ InterFnReachability->instructionCanReach(A, FnFirstInst, *RQI.To,
+ RQI.ExclusionSet))
+ return false;
+ }
+ return true;
+ };
+
+ const auto *IntraFnReachability = A.getAAFor<AAIntraFnReachability>(
+ *this, IRPosition::function(*RQI.From->getFunction()),
+ DepClassTy::OPTIONAL);
+
+ // Determine call like instructions that we can reach from the inst.
+ auto CheckCallBase = [&](Instruction &CBInst) {
+ // There are usually less nodes in the call graph, check inter function
+ // reachability first.
+ if (CheckReachableCallBase(cast<CallBase>(&CBInst)))
+ return true;
+ return IntraFnReachability && !IntraFnReachability->isAssumedReachable(
+ A, *RQI.From, CBInst, RQI.ExclusionSet);
+ };
+
+ bool UsedExclusionSet = /* conservative */ true;
+ bool UsedAssumedInformation = false;
+ if (!A.checkForAllCallLikeInstructions(CheckCallBase, *this,
+ UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ true))
+ return rememberResult(A, RQITy::Reachable::Yes, RQI, UsedExclusionSet,
+ IsTemporaryRQI);
+
+ return rememberResult(A, RQITy::Reachable::No, RQI, UsedExclusionSet,
+ IsTemporaryRQI);
+ }
+
+ void trackStatistics() const override {}
+};
+} // namespace
+
+template <typename AAType>
+static std::optional<Constant *>
+askForAssumedConstant(Attributor &A, const AbstractAttribute &QueryingAA,
+ const IRPosition &IRP, Type &Ty) {
+ if (!Ty.isIntegerTy())
+ return nullptr;
+
+ // This will also pass the call base context.
+ const auto *AA = A.getAAFor<AAType>(QueryingAA, IRP, DepClassTy::NONE);
+ if (!AA)
+ return nullptr;
+
+ std::optional<Constant *> COpt = AA->getAssumedConstant(A);
+
+ if (!COpt.has_value()) {
+ A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);
+ return std::nullopt;
+ }
+ if (auto *C = *COpt) {
+ A.recordDependence(*AA, QueryingAA, DepClassTy::OPTIONAL);
+ return C;
+ }
+ return nullptr;
+}
+
+Value *AAPotentialValues::getSingleValue(
+ Attributor &A, const AbstractAttribute &AA, const IRPosition &IRP,
+ SmallVectorImpl<AA::ValueAndContext> &Values) {
+ Type &Ty = *IRP.getAssociatedType();
+ std::optional<Value *> V;
+ for (auto &It : Values) {
+ V = AA::combineOptionalValuesInAAValueLatice(V, It.getValue(), &Ty);
+ if (V.has_value() && !*V)
+ break;
+ }
+ if (!V.has_value())
+ return UndefValue::get(&Ty);
+ return *V;
+}
+
+namespace {
+struct AAPotentialValuesImpl : AAPotentialValues {
+ using StateType = PotentialLLVMValuesState;
+
+ AAPotentialValuesImpl(const IRPosition &IRP, Attributor &A)
+ : AAPotentialValues(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(..).
+ void initialize(Attributor &A) override {
+ if (A.hasSimplificationCallback(getIRPosition())) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+ Value *Stripped = getAssociatedValue().stripPointerCasts();
+ auto *CE = dyn_cast<ConstantExpr>(Stripped);
+ if (isa<Constant>(Stripped) &&
+ (!CE || CE->getOpcode() != Instruction::ICmp)) {
+ addValue(A, getState(), *Stripped, getCtxI(), AA::AnyScope,
+ getAnchorScope());
+ indicateOptimisticFixpoint();
+ return;
+ }
+ AAPotentialValues::initialize(A);
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ std::string Str;
+ llvm::raw_string_ostream OS(Str);
+ OS << getState();
+ return OS.str();
+ }
+
+ template <typename AAType>
+ static std::optional<Value *> askOtherAA(Attributor &A,
+ const AbstractAttribute &AA,
+ const IRPosition &IRP, Type &Ty) {
+ if (isa<Constant>(IRP.getAssociatedValue()))
+ return &IRP.getAssociatedValue();
+ std::optional<Constant *> C = askForAssumedConstant<AAType>(A, AA, IRP, Ty);
+ if (!C)
+ return std::nullopt;
+ if (*C)
+ if (auto *CC = AA::getWithType(**C, Ty))
+ return CC;
+ return nullptr;
+ }
+
+ virtual void addValue(Attributor &A, StateType &State, Value &V,
+ const Instruction *CtxI, AA::ValueScope S,
+ Function *AnchorScope) const {
+
+ IRPosition ValIRP = IRPosition::value(V);
+ if (auto *CB = dyn_cast_or_null<CallBase>(CtxI)) {
+ for (const auto &U : CB->args()) {
+ if (U.get() != &V)
+ continue;
+ ValIRP = IRPosition::callsite_argument(*CB, CB->getArgOperandNo(&U));
+ break;
+ }
+ }
+
+ Value *VPtr = &V;
+ if (ValIRP.getAssociatedType()->isIntegerTy()) {
+ Type &Ty = *getAssociatedType();
+ std::optional<Value *> SimpleV =
+ askOtherAA<AAValueConstantRange>(A, *this, ValIRP, Ty);
+ if (SimpleV.has_value() && !*SimpleV) {
+ auto *PotentialConstantsAA = A.getAAFor<AAPotentialConstantValues>(
+ *this, ValIRP, DepClassTy::OPTIONAL);
+ if (PotentialConstantsAA && PotentialConstantsAA->isValidState()) {
+ for (const auto &It : PotentialConstantsAA->getAssumedSet())
+ State.unionAssumed({{*ConstantInt::get(&Ty, It), nullptr}, S});
+ if (PotentialConstantsAA->undefIsContained())
+ State.unionAssumed({{*UndefValue::get(&Ty), nullptr}, S});
+ return;
+ }
+ }
+ if (!SimpleV.has_value())
+ return;
+
+ if (*SimpleV)
+ VPtr = *SimpleV;
+ }
+
+ if (isa<ConstantInt>(VPtr))
+ CtxI = nullptr;
+ if (!AA::isValidInScope(*VPtr, AnchorScope))
+ S = AA::ValueScope(S | AA::Interprocedural);
+
+ State.unionAssumed({{*VPtr, CtxI}, S});
+ }
+
+ /// Helper struct to tie a value+context pair together with the scope for
+ /// which this is the simplified version.
+ struct ItemInfo {
+ AA::ValueAndContext I;
+ AA::ValueScope S;
+
+ bool operator==(const ItemInfo &II) const {
+ return II.I == I && II.S == S;
+ };
+ bool operator<(const ItemInfo &II) const {
+ if (I == II.I)
+ return S < II.S;
+ return I < II.I;
+ };
+ };
+
+ bool recurseForValue(Attributor &A, const IRPosition &IRP, AA::ValueScope S) {
+ SmallMapVector<AA::ValueAndContext, int, 8> ValueScopeMap;
+ for (auto CS : {AA::Intraprocedural, AA::Interprocedural}) {
+ if (!(CS & S))
+ continue;
+
+ bool UsedAssumedInformation = false;
+ SmallVector<AA::ValueAndContext> Values;
+ if (!A.getAssumedSimplifiedValues(IRP, this, Values, CS,
+ UsedAssumedInformation))
+ return false;
+
+ for (auto &It : Values)
+ ValueScopeMap[It] += CS;
+ }
+ for (auto &It : ValueScopeMap)
+ addValue(A, getState(), *It.first.getValue(), It.first.getCtxI(),
+ AA::ValueScope(It.second), getAnchorScope());
+
+ return true;
+ }
+
+ void giveUpOnIntraprocedural(Attributor &A) {
+ auto NewS = StateType::getBestState(getState());
+ for (const auto &It : getAssumedSet()) {
+ if (It.second == AA::Intraprocedural)
+ continue;
+ addValue(A, NewS, *It.first.getValue(), It.first.getCtxI(),
+ AA::Interprocedural, getAnchorScope());
+ }
+ assert(!undefIsContained() && "Undef should be an explicit value!");
+ addValue(A, NewS, getAssociatedValue(), getCtxI(), AA::Intraprocedural,
+ getAnchorScope());
+ getState() = NewS;
+ }
+
+ /// See AbstractState::indicatePessimisticFixpoint(...).
+ ChangeStatus indicatePessimisticFixpoint() override {
+ getState() = StateType::getBestState(getState());
+ getState().unionAssumed({{getAssociatedValue(), getCtxI()}, AA::AnyScope});
+ AAPotentialValues::indicateOptimisticFixpoint();
+ return ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ return indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ SmallVector<AA::ValueAndContext> Values;
+ for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
+ Values.clear();
+ if (!getAssumedSimplifiedValues(A, Values, S))
+ continue;
+ Value &OldV = getAssociatedValue();
+ if (isa<UndefValue>(OldV))
+ continue;
+ Value *NewV = getSingleValue(A, *this, getIRPosition(), Values);
+ if (!NewV || NewV == &OldV)
+ continue;
+ if (getCtxI() &&
+ !AA::isValidAtPosition({*NewV, *getCtxI()}, A.getInfoCache()))
+ continue;
+ if (A.changeAfterManifest(getIRPosition(), *NewV))
+ return ChangeStatus::CHANGED;
+ }
+ return ChangeStatus::UNCHANGED;
+ }
+
+ bool getAssumedSimplifiedValues(
+ Attributor &A, SmallVectorImpl<AA::ValueAndContext> &Values,
+ AA::ValueScope S, bool RecurseForSelectAndPHI = false) const override {
+ if (!isValidState())
+ return false;
+ bool UsedAssumedInformation = false;
+ for (const auto &It : getAssumedSet())
+ if (It.second & S) {
+ if (RecurseForSelectAndPHI && (isa<PHINode>(It.first.getValue()) ||
+ isa<SelectInst>(It.first.getValue()))) {
+ if (A.getAssumedSimplifiedValues(
+ IRPosition::inst(*cast<Instruction>(It.first.getValue())),
+ this, Values, S, UsedAssumedInformation))
+ continue;
+ }
+ Values.push_back(It.first);
+ }
+ assert(!undefIsContained() && "Undef should be an explicit value!");
+ return true;
+ }
+};
+
+struct AAPotentialValuesFloating : AAPotentialValuesImpl {
+ AAPotentialValuesFloating(const IRPosition &IRP, Attributor &A)
+ : AAPotentialValuesImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto AssumedBefore = getAssumed();
+
+ genericValueTraversal(A, &getAssociatedValue());
+
+ return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ /// Helper struct to remember which AAIsDead instances we actually used.
+ struct LivenessInfo {
+ const AAIsDead *LivenessAA = nullptr;
+ bool AnyDead = false;
+ };
+
+ /// Check if \p Cmp is a comparison we can simplify.
+ ///
+ /// We handle multiple cases, one in which at least one operand is an
+ /// (assumed) nullptr. If so, try to simplify it using AANonNull on the other
+ /// operand. Return true if successful, in that case Worklist will be updated.
+ bool handleCmp(Attributor &A, Value &Cmp, Value *LHS, Value *RHS,
+ CmpInst::Predicate Pred, ItemInfo II,
+ SmallVectorImpl<ItemInfo> &Worklist) {
+
+ // Simplify the operands first.
+ bool UsedAssumedInformation = false;
+ SmallVector<AA::ValueAndContext> LHSValues, RHSValues;
+ auto GetSimplifiedValues = [&](Value &V,
+ SmallVector<AA::ValueAndContext> &Values) {
+ if (!A.getAssumedSimplifiedValues(
+ IRPosition::value(V, getCallBaseContext()), this, Values,
+ AA::Intraprocedural, UsedAssumedInformation)) {
+ Values.clear();
+ Values.push_back(AA::ValueAndContext{V, II.I.getCtxI()});
+ }
+ return Values.empty();
+ };
+ if (GetSimplifiedValues(*LHS, LHSValues))
+ return true;
+ if (GetSimplifiedValues(*RHS, RHSValues))
+ return true;
+
+ LLVMContext &Ctx = LHS->getContext();
+
+ InformationCache &InfoCache = A.getInfoCache();
+ Instruction *CmpI = dyn_cast<Instruction>(&Cmp);
+ Function *F = CmpI ? CmpI->getFunction() : nullptr;
+ const auto *DT =
+ F ? InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F)
+ : nullptr;
+ const auto *TLI =
+ F ? A.getInfoCache().getTargetLibraryInfoForFunction(*F) : nullptr;
+ auto *AC =
+ F ? InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F)
+ : nullptr;
+
+ const DataLayout &DL = A.getDataLayout();
+ SimplifyQuery Q(DL, TLI, DT, AC, CmpI);
+
+ auto CheckPair = [&](Value &LHSV, Value &RHSV) {
+ if (isa<UndefValue>(LHSV) || isa<UndefValue>(RHSV)) {
+ addValue(A, getState(), *UndefValue::get(Cmp.getType()),
+ /* CtxI */ nullptr, II.S, getAnchorScope());
+ return true;
+ }
+
+ // Handle the trivial case first in which we don't even need to think
+ // about null or non-null.
+ if (&LHSV == &RHSV &&
+ (CmpInst::isTrueWhenEqual(Pred) || CmpInst::isFalseWhenEqual(Pred))) {
+ Constant *NewV = ConstantInt::get(Type::getInt1Ty(Ctx),
+ CmpInst::isTrueWhenEqual(Pred));
+ addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
+ getAnchorScope());
+ return true;
+ }
+
+ auto *TypedLHS = AA::getWithType(LHSV, *LHS->getType());
+ auto *TypedRHS = AA::getWithType(RHSV, *RHS->getType());
+ if (TypedLHS && TypedRHS) {
+ Value *NewV = simplifyCmpInst(Pred, TypedLHS, TypedRHS, Q);
+ if (NewV && NewV != &Cmp) {
+ addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
+ getAnchorScope());
+ return true;
+ }
+ }
+
+ // From now on we only handle equalities (==, !=).
+ if (!CmpInst::isEquality(Pred))
+ return false;
+
+ bool LHSIsNull = isa<ConstantPointerNull>(LHSV);
+ bool RHSIsNull = isa<ConstantPointerNull>(RHSV);
+ if (!LHSIsNull && !RHSIsNull)
+ return false;
+
+ // Left is the nullptr ==/!= non-nullptr case. We'll use AANonNull on the
+ // non-nullptr operand and if we assume it's non-null we can conclude the
+ // result of the comparison.
+ assert((LHSIsNull || RHSIsNull) &&
+ "Expected nullptr versus non-nullptr comparison at this point");
+
+ // The index is the operand that we assume is not null.
+ unsigned PtrIdx = LHSIsNull;
+ bool IsKnownNonNull;
+ bool IsAssumedNonNull = AA::hasAssumedIRAttr<Attribute::NonNull>(
+ A, this, IRPosition::value(*(PtrIdx ? &RHSV : &LHSV)),
+ DepClassTy::REQUIRED, IsKnownNonNull);
+ if (!IsAssumedNonNull)
+ return false;
+
+ // The new value depends on the predicate, true for != and false for ==.
+ Constant *NewV =
+ ConstantInt::get(Type::getInt1Ty(Ctx), Pred == CmpInst::ICMP_NE);
+ addValue(A, getState(), *NewV, /* CtxI */ nullptr, II.S,
+ getAnchorScope());
+ return true;
+ };
+
+ for (auto &LHSValue : LHSValues)
+ for (auto &RHSValue : RHSValues)
+ if (!CheckPair(*LHSValue.getValue(), *RHSValue.getValue()))
+ return false;
+ return true;
+ }
+
+ bool handleSelectInst(Attributor &A, SelectInst &SI, ItemInfo II,
+ SmallVectorImpl<ItemInfo> &Worklist) {
+ const Instruction *CtxI = II.I.getCtxI();
+ bool UsedAssumedInformation = false;
+
+ std::optional<Constant *> C =
+ A.getAssumedConstant(*SI.getCondition(), *this, UsedAssumedInformation);
+ bool NoValueYet = !C.has_value();
+ if (NoValueYet || isa_and_nonnull<UndefValue>(*C))
+ return true;
+ if (auto *CI = dyn_cast_or_null<ConstantInt>(*C)) {
+ if (CI->isZero())
+ Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});
+ else
+ Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});
+ } else if (&SI == &getAssociatedValue()) {
+ // We could not simplify the condition, assume both values.
+ Worklist.push_back({{*SI.getTrueValue(), CtxI}, II.S});
+ Worklist.push_back({{*SI.getFalseValue(), CtxI}, II.S});
+ } else {
+ std::optional<Value *> SimpleV = A.getAssumedSimplified(
+ IRPosition::inst(SI), *this, UsedAssumedInformation, II.S);
+ if (!SimpleV.has_value())
+ return true;
+ if (*SimpleV) {
+ addValue(A, getState(), **SimpleV, CtxI, II.S, getAnchorScope());
+ return true;
+ }
+ return false;
+ }
+ return true;
+ }
+
+ bool handleLoadInst(Attributor &A, LoadInst &LI, ItemInfo II,
+ SmallVectorImpl<ItemInfo> &Worklist) {
+ SmallSetVector<Value *, 4> PotentialCopies;
+ SmallSetVector<Instruction *, 4> PotentialValueOrigins;
+ bool UsedAssumedInformation = false;
+ if (!AA::getPotentiallyLoadedValues(A, LI, PotentialCopies,
+ PotentialValueOrigins, *this,
+ UsedAssumedInformation,
+ /* OnlyExact */ true)) {
+ LLVM_DEBUG(dbgs() << "[AAPotentialValues] Failed to get potentially "
+ "loaded values for load instruction "
+ << LI << "\n");
+ return false;
+ }
+
+ // Do not simplify loads that are only used in llvm.assume if we cannot also
+ // remove all stores that may feed into the load. The reason is that the
+ // assume is probably worth something as long as the stores are around.
+ InformationCache &InfoCache = A.getInfoCache();
+ if (InfoCache.isOnlyUsedByAssume(LI)) {
+ if (!llvm::all_of(PotentialValueOrigins, [&](Instruction *I) {
+ if (!I || isa<AssumeInst>(I))
+ return true;
+ if (auto *SI = dyn_cast<StoreInst>(I))
+ return A.isAssumedDead(SI->getOperandUse(0), this,
+ /* LivenessAA */ nullptr,
+ UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ false);
+ return A.isAssumedDead(*I, this, /* LivenessAA */ nullptr,
+ UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ false);
+ })) {
+ LLVM_DEBUG(dbgs() << "[AAPotentialValues] Load is onl used by assumes "
+ "and we cannot delete all the stores: "
+ << LI << "\n");
+ return false;
+ }
+ }
+
+ // Values have to be dynamically unique or we loose the fact that a
+ // single llvm::Value might represent two runtime values (e.g.,
+ // stack locations in different recursive calls).
+ const Instruction *CtxI = II.I.getCtxI();
+ bool ScopeIsLocal = (II.S & AA::Intraprocedural);
+ bool AllLocal = ScopeIsLocal;
+ bool DynamicallyUnique = llvm::all_of(PotentialCopies, [&](Value *PC) {
+ AllLocal &= AA::isValidInScope(*PC, getAnchorScope());
+ return AA::isDynamicallyUnique(A, *this, *PC);
+ });
+ if (!DynamicallyUnique) {
+ LLVM_DEBUG(dbgs() << "[AAPotentialValues] Not all potentially loaded "
+ "values are dynamically unique: "
+ << LI << "\n");
+ return false;
+ }
+
+ for (auto *PotentialCopy : PotentialCopies) {
+ if (AllLocal) {
+ Worklist.push_back({{*PotentialCopy, CtxI}, II.S});
+ } else {
+ Worklist.push_back({{*PotentialCopy, CtxI}, AA::Interprocedural});
+ }
+ }
+ if (!AllLocal && ScopeIsLocal)
+ addValue(A, getState(), LI, CtxI, AA::Intraprocedural, getAnchorScope());
+ return true;
+ }
+
+ bool handlePHINode(
+ Attributor &A, PHINode &PHI, ItemInfo II,
+ SmallVectorImpl<ItemInfo> &Worklist,
+ SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
+ auto GetLivenessInfo = [&](const Function &F) -> LivenessInfo & {
+ LivenessInfo &LI = LivenessAAs[&F];
+ if (!LI.LivenessAA)
+ LI.LivenessAA = A.getAAFor<AAIsDead>(*this, IRPosition::function(F),
+ DepClassTy::NONE);
+ return LI;
+ };
+
+ if (&PHI == &getAssociatedValue()) {
+ LivenessInfo &LI = GetLivenessInfo(*PHI.getFunction());
+ const auto *CI =
+ A.getInfoCache().getAnalysisResultForFunction<CycleAnalysis>(
+ *PHI.getFunction());
+
+ Cycle *C = nullptr;
+ bool CyclePHI = mayBeInCycle(CI, &PHI, /* HeaderOnly */ true, &C);
+ for (unsigned u = 0, e = PHI.getNumIncomingValues(); u < e; u++) {
+ BasicBlock *IncomingBB = PHI.getIncomingBlock(u);
+ if (LI.LivenessAA &&
+ LI.LivenessAA->isEdgeDead(IncomingBB, PHI.getParent())) {
+ LI.AnyDead = true;
+ continue;
+ }
+ Value *V = PHI.getIncomingValue(u);
+ if (V == &PHI)
+ continue;
+
+ // If the incoming value is not the PHI but an instruction in the same
+ // cycle we might have multiple versions of it flying around.
+ if (CyclePHI && isa<Instruction>(V) &&
+ (!C || C->contains(cast<Instruction>(V)->getParent())))
+ return false;
+
+ Worklist.push_back({{*V, IncomingBB->getTerminator()}, II.S});
+ }
+ return true;
+ }
+
+ bool UsedAssumedInformation = false;
+ std::optional<Value *> SimpleV = A.getAssumedSimplified(
+ IRPosition::inst(PHI), *this, UsedAssumedInformation, II.S);
+ if (!SimpleV.has_value())
+ return true;
+ if (!(*SimpleV))
+ return false;
+ addValue(A, getState(), **SimpleV, &PHI, II.S, getAnchorScope());
+ return true;
+ }
+
+ /// Use the generic, non-optimistic InstSimplfy functionality if we managed to
+ /// simplify any operand of the instruction \p I. Return true if successful,
+ /// in that case Worklist will be updated.
+ bool handleGenericInst(Attributor &A, Instruction &I, ItemInfo II,
+ SmallVectorImpl<ItemInfo> &Worklist) {
+ bool SomeSimplified = false;
+ bool UsedAssumedInformation = false;
+
+ SmallVector<Value *, 8> NewOps(I.getNumOperands());
+ int Idx = 0;
+ for (Value *Op : I.operands()) {
+ const auto &SimplifiedOp = A.getAssumedSimplified(
+ IRPosition::value(*Op, getCallBaseContext()), *this,
+ UsedAssumedInformation, AA::Intraprocedural);
+ // If we are not sure about any operand we are not sure about the entire
+ // instruction, we'll wait.
+ if (!SimplifiedOp.has_value())
+ return true;
+
+ if (*SimplifiedOp)
+ NewOps[Idx] = *SimplifiedOp;
+ else
+ NewOps[Idx] = Op;
+
+ SomeSimplified |= (NewOps[Idx] != Op);
+ ++Idx;
+ }
+
+ // We won't bother with the InstSimplify interface if we didn't simplify any
+ // operand ourselves.
+ if (!SomeSimplified)
+ return false;
+
+ InformationCache &InfoCache = A.getInfoCache();
+ Function *F = I.getFunction();
+ const auto *DT =
+ InfoCache.getAnalysisResultForFunction<DominatorTreeAnalysis>(*F);
+ const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
+ auto *AC = InfoCache.getAnalysisResultForFunction<AssumptionAnalysis>(*F);
+
+ const DataLayout &DL = I.getModule()->getDataLayout();
+ SimplifyQuery Q(DL, TLI, DT, AC, &I);
+ Value *NewV = simplifyInstructionWithOperands(&I, NewOps, Q);
+ if (!NewV || NewV == &I)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "Generic inst " << I << " assumed simplified to "
+ << *NewV << "\n");
+ Worklist.push_back({{*NewV, II.I.getCtxI()}, II.S});
+ return true;
+ }
+
+ bool simplifyInstruction(
+ Attributor &A, Instruction &I, ItemInfo II,
+ SmallVectorImpl<ItemInfo> &Worklist,
+ SmallMapVector<const Function *, LivenessInfo, 4> &LivenessAAs) {
+ if (auto *CI = dyn_cast<CmpInst>(&I))
+ return handleCmp(A, *CI, CI->getOperand(0), CI->getOperand(1),
+ CI->getPredicate(), II, Worklist);
+
+ switch (I.getOpcode()) {
+ case Instruction::Select:
+ return handleSelectInst(A, cast<SelectInst>(I), II, Worklist);
+ case Instruction::PHI:
+ return handlePHINode(A, cast<PHINode>(I), II, Worklist, LivenessAAs);
+ case Instruction::Load:
+ return handleLoadInst(A, cast<LoadInst>(I), II, Worklist);
+ default:
+ return handleGenericInst(A, I, II, Worklist);
+ };
+ return false;
+ }
+
+ void genericValueTraversal(Attributor &A, Value *InitialV) {
+ SmallMapVector<const Function *, LivenessInfo, 4> LivenessAAs;
+
+ SmallSet<ItemInfo, 16> Visited;
+ SmallVector<ItemInfo, 16> Worklist;
+ Worklist.push_back({{*InitialV, getCtxI()}, AA::AnyScope});
+
+ int Iteration = 0;
+ do {
+ ItemInfo II = Worklist.pop_back_val();
+ Value *V = II.I.getValue();
+ assert(V);
+ const Instruction *CtxI = II.I.getCtxI();
+ AA::ValueScope S = II.S;
+
+ // Check if we should process the current value. To prevent endless
+ // recursion keep a record of the values we followed!
+ if (!Visited.insert(II).second)
+ continue;
+
+ // Make sure we limit the compile time for complex expressions.
+ if (Iteration++ >= MaxPotentialValuesIterations) {
+ LLVM_DEBUG(dbgs() << "Generic value traversal reached iteration limit: "
+ << Iteration << "!\n");
+ addValue(A, getState(), *V, CtxI, S, getAnchorScope());
+ continue;
+ }
+
+ // Explicitly look through calls with a "returned" attribute if we do
+ // not have a pointer as stripPointerCasts only works on them.
+ Value *NewV = nullptr;
+ if (V->getType()->isPointerTy()) {
+ NewV = AA::getWithType(*V->stripPointerCasts(), *V->getType());
+ } else {
+ if (auto *CB = dyn_cast<CallBase>(V))
+ if (auto *Callee =
+ dyn_cast_if_present<Function>(CB->getCalledOperand())) {
+ for (Argument &Arg : Callee->args())
+ if (Arg.hasReturnedAttr()) {
+ NewV = CB->getArgOperand(Arg.getArgNo());
+ break;
+ }
+ }
+ }
+ if (NewV && NewV != V) {
+ Worklist.push_back({{*NewV, CtxI}, S});
+ continue;
+ }
+
+ if (auto *CE = dyn_cast<ConstantExpr>(V)) {
+ if (CE->getOpcode() == Instruction::ICmp)
+ if (handleCmp(A, *CE, CE->getOperand(0), CE->getOperand(1),
+ CmpInst::Predicate(CE->getPredicate()), II, Worklist))
+ continue;
+ }
+
+ if (auto *I = dyn_cast<Instruction>(V)) {
+ if (simplifyInstruction(A, *I, II, Worklist, LivenessAAs))
+ continue;
+ }
+
+ if (V != InitialV || isa<Argument>(V))
+ if (recurseForValue(A, IRPosition::value(*V), II.S))
+ continue;
+
+ // If we haven't stripped anything we give up.
+ if (V == InitialV && CtxI == getCtxI()) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+
+ addValue(A, getState(), *V, CtxI, S, getAnchorScope());
+ } while (!Worklist.empty());
+
+ // If we actually used liveness information so we have to record a
+ // dependence.
+ for (auto &It : LivenessAAs)
+ if (It.second.AnyDead)
+ A.recordDependence(*It.second.LivenessAA, *this, DepClassTy::OPTIONAL);
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialValuesArgument final : AAPotentialValuesImpl {
+ using Base = AAPotentialValuesImpl;
+ AAPotentialValuesArgument(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(..).
+ void initialize(Attributor &A) override {
+ auto &Arg = cast<Argument>(getAssociatedValue());
+ if (Arg.hasPointeeInMemoryValueAttr())
+ indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto AssumedBefore = getAssumed();
+
+ unsigned ArgNo = getCalleeArgNo();
+
+ bool UsedAssumedInformation = false;
+ SmallVector<AA::ValueAndContext> Values;
+ auto CallSitePred = [&](AbstractCallSite ACS) {
+ const auto CSArgIRP = IRPosition::callsite_argument(ACS, ArgNo);
+ if (CSArgIRP.getPositionKind() == IRP_INVALID)
+ return false;
+
+ if (!A.getAssumedSimplifiedValues(CSArgIRP, this, Values,
+ AA::Interprocedural,
+ UsedAssumedInformation))
+ return false;
+
+ return isValidState();
+ };
+
+ if (!A.checkForAllCallSites(CallSitePred, *this,
+ /* RequireAllCallSites */ true,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ Function *Fn = getAssociatedFunction();
+ bool AnyNonLocal = false;
+ for (auto &It : Values) {
+ if (isa<Constant>(It.getValue())) {
+ addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,
+ getAnchorScope());
+ continue;
+ }
+ if (!AA::isDynamicallyUnique(A, *this, *It.getValue()))
+ return indicatePessimisticFixpoint();
+
+ if (auto *Arg = dyn_cast<Argument>(It.getValue()))
+ if (Arg->getParent() == Fn) {
+ addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::AnyScope,
+ getAnchorScope());
+ continue;
+ }
+ addValue(A, getState(), *It.getValue(), It.getCtxI(), AA::Interprocedural,
+ getAnchorScope());
+ AnyNonLocal = true;
+ }
+ assert(!undefIsContained() && "Undef should be an explicit value!");
+ if (AnyNonLocal)
+ giveUpOnIntraprocedural(A);
+
+ return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_ARG_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialValuesReturned : public AAPotentialValuesFloating {
+ using Base = AAPotentialValuesFloating;
+ AAPotentialValuesReturned(const IRPosition &IRP, Attributor &A)
+ : Base(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(..).
+ void initialize(Attributor &A) override {
+ Function *F = getAssociatedFunction();
+ if (!F || F->isDeclaration() || F->getReturnType()->isVoidTy()) {
+ indicatePessimisticFixpoint();
+ return;
+ }
+
+ for (Argument &Arg : F->args())
+ if (Arg.hasReturnedAttr()) {
+ addValue(A, getState(), Arg, nullptr, AA::AnyScope, F);
+ ReturnedArg = &Arg;
+ break;
+ }
+ if (!A.isFunctionIPOAmendable(*F) ||
+ A.hasSimplificationCallback(getIRPosition())) {
+ if (!ReturnedArg)
+ indicatePessimisticFixpoint();
+ else
+ indicateOptimisticFixpoint();
+ }
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto AssumedBefore = getAssumed();
+ bool UsedAssumedInformation = false;
+
+ SmallVector<AA::ValueAndContext> Values;
+ Function *AnchorScope = getAnchorScope();
+ auto HandleReturnedValue = [&](Value &V, Instruction *CtxI,
+ bool AddValues) {
+ for (AA::ValueScope S : {AA::Interprocedural, AA::Intraprocedural}) {
+ Values.clear();
+ if (!A.getAssumedSimplifiedValues(IRPosition::value(V), this, Values, S,
+ UsedAssumedInformation,
+ /* RecurseForSelectAndPHI */ true))
+ return false;
+ if (!AddValues)
+ continue;
+ for (const AA::ValueAndContext &VAC : Values)
+ addValue(A, getState(), *VAC.getValue(),
+ VAC.getCtxI() ? VAC.getCtxI() : CtxI, S, AnchorScope);
+ }
+ return true;
+ };
+
+ if (ReturnedArg) {
+ HandleReturnedValue(*ReturnedArg, nullptr, true);
+ } else {
+ auto RetInstPred = [&](Instruction &RetI) {
+ bool AddValues = true;
+ if (isa<PHINode>(RetI.getOperand(0)) ||
+ isa<SelectInst>(RetI.getOperand(0))) {
+ addValue(A, getState(), *RetI.getOperand(0), &RetI, AA::AnyScope,
+ AnchorScope);
+ AddValues = false;
+ }
+ return HandleReturnedValue(*RetI.getOperand(0), &RetI, AddValues);
+ };
+
+ if (!A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
+ UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ true))
+ return indicatePessimisticFixpoint();
+ }
+
+ return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ void addValue(Attributor &A, StateType &State, Value &V,
+ const Instruction *CtxI, AA::ValueScope S,
+ Function *AnchorScope) const override {
+ Function *F = getAssociatedFunction();
+ if (auto *CB = dyn_cast<CallBase>(&V))
+ if (CB->getCalledOperand() == F)
+ return;
+ Base::addValue(A, State, V, CtxI, S, AnchorScope);
+ }
+
+ ChangeStatus manifest(Attributor &A) override {
+ if (ReturnedArg)
+ return ChangeStatus::UNCHANGED;
+ SmallVector<AA::ValueAndContext> Values;
+ if (!getAssumedSimplifiedValues(A, Values, AA::ValueScope::Intraprocedural,
+ /* RecurseForSelectAndPHI */ true))
+ return ChangeStatus::UNCHANGED;
+ Value *NewVal = getSingleValue(A, *this, getIRPosition(), Values);
+ if (!NewVal)
+ return ChangeStatus::UNCHANGED;
+
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ if (auto *Arg = dyn_cast<Argument>(NewVal)) {
+ STATS_DECLTRACK(UniqueReturnValue, FunctionReturn,
+ "Number of function with unique return");
+ Changed |= A.manifestAttrs(
+ IRPosition::argument(*Arg),
+ {Attribute::get(Arg->getContext(), Attribute::Returned)});
+ STATS_DECLTRACK_ARG_ATTR(returned);
+ }
+
+ auto RetInstPred = [&](Instruction &RetI) {
+ Value *RetOp = RetI.getOperand(0);
+ if (isa<UndefValue>(RetOp) || RetOp == NewVal)
+ return true;
+ if (AA::isValidAtPosition({*NewVal, RetI}, A.getInfoCache()))
+ if (A.changeUseAfterManifest(RetI.getOperandUse(0), *NewVal))
+ Changed = ChangeStatus::CHANGED;
+ return true;
+ };
+ bool UsedAssumedInformation = false;
+ (void)A.checkForAllInstructions(RetInstPred, *this, {Instruction::Ret},
+ UsedAssumedInformation,
+ /* CheckBBLivenessOnly */ true);
+ return Changed;
+ }
+
+ ChangeStatus indicatePessimisticFixpoint() override {
+ return AAPotentialValues::indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override{
+ STATS_DECLTRACK_FNRET_ATTR(potential_values)}
+
+ /// The argumented with an existing `returned` attribute.
+ Argument *ReturnedArg = nullptr;
+};
+
+struct AAPotentialValuesFunction : AAPotentialValuesImpl {
+ AAPotentialValuesFunction(const IRPosition &IRP, Attributor &A)
+ : AAPotentialValuesImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ llvm_unreachable("AAPotentialValues(Function|CallSite)::updateImpl will "
+ "not be called");
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FN_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialValuesCallSite : AAPotentialValuesFunction {
+ AAPotentialValuesCallSite(const IRPosition &IRP, Attributor &A)
+ : AAPotentialValuesFunction(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CS_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialValuesCallSiteReturned : AAPotentialValuesImpl {
+ AAPotentialValuesCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AAPotentialValuesImpl(IRP, A) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto AssumedBefore = getAssumed();
+
+ Function *Callee = getAssociatedFunction();
+ if (!Callee)
+ return indicatePessimisticFixpoint();
+
+ bool UsedAssumedInformation = false;
+ auto *CB = cast<CallBase>(getCtxI());
+ if (CB->isMustTailCall() &&
+ !A.isAssumedDead(IRPosition::inst(*CB), this, nullptr,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ SmallVector<AA::ValueAndContext> Values;
+ if (!A.getAssumedSimplifiedValues(IRPosition::returned(*Callee), this,
+ Values, AA::Intraprocedural,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ Function *Caller = CB->getCaller();
+
+ bool AnyNonLocal = false;
+ for (auto &It : Values) {
+ Value *V = It.getValue();
+ std::optional<Value *> CallerV = A.translateArgumentToCallSiteContent(
+ V, *CB, *this, UsedAssumedInformation);
+ if (!CallerV.has_value()) {
+ // Nothing to do as long as no value was determined.
+ continue;
+ }
+ V = *CallerV ? *CallerV : V;
+ if (AA::isDynamicallyUnique(A, *this, *V) &&
+ AA::isValidInScope(*V, Caller)) {
+ if (*CallerV) {
+ SmallVector<AA::ValueAndContext> ArgValues;
+ IRPosition IRP = IRPosition::value(*V);
+ if (auto *Arg = dyn_cast<Argument>(V))
+ if (Arg->getParent() == CB->getCalledOperand())
+ IRP = IRPosition::callsite_argument(*CB, Arg->getArgNo());
+ if (recurseForValue(A, IRP, AA::AnyScope))
+ continue;
+ }
+ addValue(A, getState(), *V, CB, AA::AnyScope, getAnchorScope());
+ } else {
+ AnyNonLocal = true;
+ break;
+ }
+ }
+ if (AnyNonLocal) {
+ Values.clear();
+ if (!A.getAssumedSimplifiedValues(IRPosition::returned(*Callee), this,
+ Values, AA::Interprocedural,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+ AnyNonLocal = false;
+ getState() = PotentialLLVMValuesState::getBestState();
+ for (auto &It : Values) {
+ Value *V = It.getValue();
+ if (!AA::isDynamicallyUnique(A, *this, *V))
+ return indicatePessimisticFixpoint();
+ if (AA::isValidInScope(*V, Caller)) {
+ addValue(A, getState(), *V, CB, AA::AnyScope, getAnchorScope());
+ } else {
+ AnyNonLocal = true;
+ addValue(A, getState(), *V, CB, AA::Interprocedural,
+ getAnchorScope());
+ }
+ }
+ if (AnyNonLocal)
+ giveUpOnIntraprocedural(A);
+ }
+ return (AssumedBefore == getAssumed()) ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ ChangeStatus indicatePessimisticFixpoint() override {
+ return AAPotentialValues::indicatePessimisticFixpoint();
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSRET_ATTR(potential_values)
+ }
+};
+
+struct AAPotentialValuesCallSiteArgument : AAPotentialValuesFloating {
+ AAPotentialValuesCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAPotentialValuesFloating(IRP, A) {}
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSARG_ATTR(potential_values)
+ }
+};
+} // namespace
+
+/// ---------------------- Assumption Propagation ------------------------------
+namespace {
+struct AAAssumptionInfoImpl : public AAAssumptionInfo {
+ AAAssumptionInfoImpl(const IRPosition &IRP, Attributor &A,
+ const DenseSet<StringRef> &Known)
+ : AAAssumptionInfo(IRP, A, Known) {}
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ // Don't manifest a universal set if it somehow made it here.
+ if (getKnown().isUniversal())
+ return ChangeStatus::UNCHANGED;
+
+ const IRPosition &IRP = getIRPosition();
+ return A.manifestAttrs(
+ IRP,
+ Attribute::get(IRP.getAnchorValue().getContext(), AssumptionAttrKey,
+ llvm::join(getAssumed().getSet(), ",")),
+ /* ForceReplace */ true);
+ }
+
+ bool hasAssumption(const StringRef Assumption) const override {
+ return isValidState() && setContains(Assumption);
+ }
+
+ /// See AbstractAttribute::getAsStr()
+ const std::string getAsStr(Attributor *A) const override {
+ const SetContents &Known = getKnown();
+ const SetContents &Assumed = getAssumed();
+
+ const std::string KnownStr =
+ llvm::join(Known.getSet().begin(), Known.getSet().end(), ",");
+ const std::string AssumedStr =
+ (Assumed.isUniversal())
+ ? "Universal"
+ : llvm::join(Assumed.getSet().begin(), Assumed.getSet().end(), ",");
+
+ return "Known [" + KnownStr + "]," + " Assumed [" + AssumedStr + "]";
+ }
+};
+
+/// Propagates assumption information from parent functions to all of their
+/// successors. An assumption can be propagated if the containing function
+/// dominates the called function.
+///
+/// We start with a "known" set of assumptions already valid for the associated
+/// function and an "assumed" set that initially contains all possible
+/// assumptions. The assumed set is inter-procedurally updated by narrowing its
+/// contents as concrete values are known. The concrete values are seeded by the
+/// first nodes that are either entries into the call graph, or contains no
+/// assumptions. Each node is updated as the intersection of the assumed state
+/// with all of its predecessors.
+struct AAAssumptionInfoFunction final : AAAssumptionInfoImpl {
+ AAAssumptionInfoFunction(const IRPosition &IRP, Attributor &A)
+ : AAAssumptionInfoImpl(IRP, A,
+ getAssumptions(*IRP.getAssociatedFunction())) {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ bool Changed = false;
+
+ auto CallSitePred = [&](AbstractCallSite ACS) {
+ const auto *AssumptionAA = A.getAAFor<AAAssumptionInfo>(
+ *this, IRPosition::callsite_function(*ACS.getInstruction()),
+ DepClassTy::REQUIRED);
+ if (!AssumptionAA)
+ return false;
+ // Get the set of assumptions shared by all of this function's callers.
+ Changed |= getIntersection(AssumptionAA->getAssumed());
+ return !getAssumed().empty() || !getKnown().empty();
+ };
+
+ bool UsedAssumedInformation = false;
+ // Get the intersection of all assumptions held by this node's predecessors.
+ // If we don't know all the call sites then this is either an entry into the
+ // call graph or an empty node. This node is known to only contain its own
+ // assumptions and can be propagated to its successors.
+ if (!A.checkForAllCallSites(CallSitePred, *this, true,
+ UsedAssumedInformation))
+ return indicatePessimisticFixpoint();
+
+ return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
+ }
+
+ void trackStatistics() const override {}
+};
+
+/// Assumption Info defined for call sites.
+struct AAAssumptionInfoCallSite final : AAAssumptionInfoImpl {
+
+ AAAssumptionInfoCallSite(const IRPosition &IRP, Attributor &A)
+ : AAAssumptionInfoImpl(IRP, A, getInitialAssumptions(IRP)) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
+ A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);
+ }
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ const IRPosition &FnPos = IRPosition::function(*getAnchorScope());
+ auto *AssumptionAA =
+ A.getAAFor<AAAssumptionInfo>(*this, FnPos, DepClassTy::REQUIRED);
+ if (!AssumptionAA)
+ return indicatePessimisticFixpoint();
+ bool Changed = getIntersection(AssumptionAA->getAssumed());
+ return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {}
+
+private:
+ /// Helper to initialized the known set as all the assumptions this call and
+ /// the callee contain.
+ DenseSet<StringRef> getInitialAssumptions(const IRPosition &IRP) {
+ const CallBase &CB = cast<CallBase>(IRP.getAssociatedValue());
+ auto Assumptions = getAssumptions(CB);
+ if (const Function *F = CB.getCaller())
+ set_union(Assumptions, getAssumptions(*F));
+ if (Function *F = IRP.getAssociatedFunction())
+ set_union(Assumptions, getAssumptions(*F));
+ return Assumptions;
+ }
+};
+} // namespace
+
+AACallGraphNode *AACallEdgeIterator::operator*() const {
+ return static_cast<AACallGraphNode *>(const_cast<AACallEdges *>(
+ A.getOrCreateAAFor<AACallEdges>(IRPosition::function(**I))));
+}
+
+void AttributorCallGraph::print() { llvm::WriteGraph(outs(), this); }
+
+/// ------------------------ UnderlyingObjects ---------------------------------
+
+namespace {
+struct AAUnderlyingObjectsImpl
+ : StateWrapper<BooleanState, AAUnderlyingObjects> {
+ using BaseTy = StateWrapper<BooleanState, AAUnderlyingObjects>;
+ AAUnderlyingObjectsImpl(const IRPosition &IRP, Attributor &A) : BaseTy(IRP) {}
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return std::string("UnderlyingObjects ") +
+ (isValidState()
+ ? (std::string("inter #") +
+ std::to_string(InterAssumedUnderlyingObjects.size()) +
+ " objs" + std::string(", intra #") +
+ std::to_string(IntraAssumedUnderlyingObjects.size()) +
+ " objs")
+ : "<invalid>");
+ }
+
+ /// See AbstractAttribute::trackStatistics()
+ void trackStatistics() const override {}
+
+ /// See AbstractAttribute::updateImpl(...).
+ ChangeStatus updateImpl(Attributor &A) override {
+ auto &Ptr = getAssociatedValue();
+
+ auto DoUpdate = [&](SmallSetVector<Value *, 8> &UnderlyingObjects,
+ AA::ValueScope Scope) {
+ bool UsedAssumedInformation = false;
+ SmallPtrSet<Value *, 8> SeenObjects;
+ SmallVector<AA::ValueAndContext> Values;
+
+ if (!A.getAssumedSimplifiedValues(IRPosition::value(Ptr), *this, Values,
+ Scope, UsedAssumedInformation))
+ return UnderlyingObjects.insert(&Ptr);
+
+ bool Changed = false;
+
+ for (unsigned I = 0; I < Values.size(); ++I) {
+ auto &VAC = Values[I];
+ auto *Obj = VAC.getValue();
+ Value *UO = getUnderlyingObject(Obj);
+ if (UO && UO != VAC.getValue() && SeenObjects.insert(UO).second) {
+ const auto *OtherAA = A.getAAFor<AAUnderlyingObjects>(
+ *this, IRPosition::value(*UO), DepClassTy::OPTIONAL);
+ auto Pred = [&Values](Value &V) {
+ Values.emplace_back(V, nullptr);
+ return true;
+ };
+
+ if (!OtherAA || !OtherAA->forallUnderlyingObjects(Pred, Scope))
+ llvm_unreachable(
+ "The forall call should not return false at this position");
+
+ continue;
+ }
+
+ if (isa<SelectInst>(Obj)) {
+ Changed |= handleIndirect(A, *Obj, UnderlyingObjects, Scope);
+ continue;
+ }
+ if (auto *PHI = dyn_cast<PHINode>(Obj)) {
+ // Explicitly look through PHIs as we do not care about dynamically
+ // uniqueness.
+ for (unsigned u = 0, e = PHI->getNumIncomingValues(); u < e; u++) {
+ Changed |= handleIndirect(A, *PHI->getIncomingValue(u),
+ UnderlyingObjects, Scope);
+ }
+ continue;
+ }
+
+ Changed |= UnderlyingObjects.insert(Obj);
+ }
+
+ return Changed;
+ };
+
+ bool Changed = false;
+ Changed |= DoUpdate(IntraAssumedUnderlyingObjects, AA::Intraprocedural);
+ Changed |= DoUpdate(InterAssumedUnderlyingObjects, AA::Interprocedural);
+
+ return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
+ }
+
+ bool forallUnderlyingObjects(
+ function_ref<bool(Value &)> Pred,
+ AA::ValueScope Scope = AA::Interprocedural) const override {
+ if (!isValidState())
+ return Pred(getAssociatedValue());
+
+ auto &AssumedUnderlyingObjects = Scope == AA::Intraprocedural
+ ? IntraAssumedUnderlyingObjects
+ : InterAssumedUnderlyingObjects;
+ for (Value *Obj : AssumedUnderlyingObjects)
+ if (!Pred(*Obj))
+ return false;
+
+ return true;
+ }
+
+private:
+ /// Handle the case where the value is not the actual underlying value, such
+ /// as a phi node or a select instruction.
+ bool handleIndirect(Attributor &A, Value &V,
+ SmallSetVector<Value *, 8> &UnderlyingObjects,
+ AA::ValueScope Scope) {
+ bool Changed = false;
+ const auto *AA = A.getAAFor<AAUnderlyingObjects>(
+ *this, IRPosition::value(V), DepClassTy::OPTIONAL);
+ auto Pred = [&](Value &V) {
+ Changed |= UnderlyingObjects.insert(&V);
+ return true;
+ };
+ if (!AA || !AA->forallUnderlyingObjects(Pred, Scope))
+ llvm_unreachable(
+ "The forall call should not return false at this position");
+ return Changed;
+ }
+
+ /// All the underlying objects collected so far via intra procedural scope.
+ SmallSetVector<Value *, 8> IntraAssumedUnderlyingObjects;
+ /// All the underlying objects collected so far via inter procedural scope.
+ SmallSetVector<Value *, 8> InterAssumedUnderlyingObjects;
+};
+
+struct AAUnderlyingObjectsFloating final : AAUnderlyingObjectsImpl {
+ AAUnderlyingObjectsFloating(const IRPosition &IRP, Attributor &A)
+ : AAUnderlyingObjectsImpl(IRP, A) {}
+};
+
+struct AAUnderlyingObjectsArgument final : AAUnderlyingObjectsImpl {
+ AAUnderlyingObjectsArgument(const IRPosition &IRP, Attributor &A)
+ : AAUnderlyingObjectsImpl(IRP, A) {}
+};
+
+struct AAUnderlyingObjectsCallSite final : AAUnderlyingObjectsImpl {
+ AAUnderlyingObjectsCallSite(const IRPosition &IRP, Attributor &A)
+ : AAUnderlyingObjectsImpl(IRP, A) {}
+};
+
+struct AAUnderlyingObjectsCallSiteArgument final : AAUnderlyingObjectsImpl {
+ AAUnderlyingObjectsCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAUnderlyingObjectsImpl(IRP, A) {}
+};
+
+struct AAUnderlyingObjectsReturned final : AAUnderlyingObjectsImpl {
+ AAUnderlyingObjectsReturned(const IRPosition &IRP, Attributor &A)
+ : AAUnderlyingObjectsImpl(IRP, A) {}
+};
+
+struct AAUnderlyingObjectsCallSiteReturned final : AAUnderlyingObjectsImpl {
+ AAUnderlyingObjectsCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AAUnderlyingObjectsImpl(IRP, A) {}
+};
+
+struct AAUnderlyingObjectsFunction final : AAUnderlyingObjectsImpl {
+ AAUnderlyingObjectsFunction(const IRPosition &IRP, Attributor &A)
+ : AAUnderlyingObjectsImpl(IRP, A) {}
+};
+} // namespace
+
+/// ------------------------ Global Value Info -------------------------------
+namespace {
+struct AAGlobalValueInfoFloating : public AAGlobalValueInfo {
+ AAGlobalValueInfoFloating(const IRPosition &IRP, Attributor &A)
+ : AAGlobalValueInfo(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {}
+
+ bool checkUse(Attributor &A, const Use &U, bool &Follow,
+ SmallVectorImpl<const Value *> &Worklist) {
+ Instruction *UInst = dyn_cast<Instruction>(U.getUser());
+ if (!UInst) {
+ Follow = true;
+ return true;
+ }
+
+ LLVM_DEBUG(dbgs() << "[AAGlobalValueInfo] Check use: " << *U.get() << " in "
+ << *UInst << "\n");
+
+ if (auto *Cmp = dyn_cast<ICmpInst>(U.getUser())) {
+ int Idx = &Cmp->getOperandUse(0) == &U;
+ if (isa<Constant>(Cmp->getOperand(Idx)))
+ return true;
+ return U == &getAnchorValue();
+ }
+
+ // Explicitly catch return instructions.
+ if (isa<ReturnInst>(UInst)) {
+ auto CallSitePred = [&](AbstractCallSite ACS) {
+ Worklist.push_back(ACS.getInstruction());
+ return true;
+ };
+ bool UsedAssumedInformation = false;
+ // TODO: We should traverse the uses or add a "non-call-site" CB.
+ if (!A.checkForAllCallSites(CallSitePred, *UInst->getFunction(),
+ /*RequireAllCallSites=*/true, this,
+ UsedAssumedInformation))
+ return false;
+ return true;
+ }
+
+ // For now we only use special logic for call sites. However, the tracker
+ // itself knows about a lot of other non-capturing cases already.
+ auto *CB = dyn_cast<CallBase>(UInst);
+ if (!CB)
+ return false;
+ // Direct calls are OK uses.
+ if (CB->isCallee(&U))
+ return true;
+ // Non-argument uses are scary.
+ if (!CB->isArgOperand(&U))
+ return false;
+ // TODO: Iterate callees.
+ auto *Fn = dyn_cast<Function>(CB->getCalledOperand());
+ if (!Fn || !A.isFunctionIPOAmendable(*Fn))
+ return false;
+
+ unsigned ArgNo = CB->getArgOperandNo(&U);
+ Worklist.push_back(Fn->getArg(ArgNo));
+ return true;
+ }
+
+ ChangeStatus updateImpl(Attributor &A) override {
+ unsigned NumUsesBefore = Uses.size();
+
+ SmallPtrSet<const Value *, 8> Visited;
+ SmallVector<const Value *> Worklist;
+ Worklist.push_back(&getAnchorValue());
+
+ auto UsePred = [&](const Use &U, bool &Follow) -> bool {
+ Uses.insert(&U);
+ switch (DetermineUseCaptureKind(U, nullptr)) {
+ case UseCaptureKind::NO_CAPTURE:
+ return checkUse(A, U, Follow, Worklist);
+ case UseCaptureKind::MAY_CAPTURE:
+ return checkUse(A, U, Follow, Worklist);
+ case UseCaptureKind::PASSTHROUGH:
+ Follow = true;
+ return true;
+ }
+ return true;
+ };
+ auto EquivalentUseCB = [&](const Use &OldU, const Use &NewU) {
+ Uses.insert(&OldU);
+ return true;
+ };
+
+ while (!Worklist.empty()) {
+ const Value *V = Worklist.pop_back_val();
+ if (!Visited.insert(V).second)
+ continue;
+ if (!A.checkForAllUses(UsePred, *this, *V,
+ /* CheckBBLivenessOnly */ true,
+ DepClassTy::OPTIONAL,
+ /* IgnoreDroppableUses */ true, EquivalentUseCB)) {
+ return indicatePessimisticFixpoint();
+ }
+ }
+
+ return Uses.size() == NumUsesBefore ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ bool isPotentialUse(const Use &U) const override {
+ return !isValidState() || Uses.contains(&U);
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ return ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return "[" + std::to_string(Uses.size()) + " uses]";
+ }
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(GlobalValuesTracked);
+ }
+
+private:
+ /// Set of (transitive) uses of this GlobalValue.
+ SmallPtrSet<const Use *, 8> Uses;
+};
+} // namespace
+
+/// ------------------------ Indirect Call Info -------------------------------
+namespace {
+struct AAIndirectCallInfoCallSite : public AAIndirectCallInfo {
+ AAIndirectCallInfoCallSite(const IRPosition &IRP, Attributor &A)
+ : AAIndirectCallInfo(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ auto *MD = getCtxI()->getMetadata(LLVMContext::MD_callees);
+ if (!MD && !A.isClosedWorldModule())
+ return;
+
+ if (MD) {
+ for (const auto &Op : MD->operands())
+ if (Function *Callee = mdconst::dyn_extract_or_null<Function>(Op))
+ PotentialCallees.insert(Callee);
+ } else if (A.isClosedWorldModule()) {
+ ArrayRef<Function *> IndirectlyCallableFunctions =
+ A.getInfoCache().getIndirectlyCallableFunctions(A);
+ PotentialCallees.insert(IndirectlyCallableFunctions.begin(),
+ IndirectlyCallableFunctions.end());
+ }
+
+ if (PotentialCallees.empty())
+ indicateOptimisticFixpoint();
+ }
+
+ ChangeStatus updateImpl(Attributor &A) override {
+ CallBase *CB = cast<CallBase>(getCtxI());
+ const Use &CalleeUse = CB->getCalledOperandUse();
+ Value *FP = CB->getCalledOperand();
+
+ SmallSetVector<Function *, 4> AssumedCalleesNow;
+ bool AllCalleesKnownNow = AllCalleesKnown;
+
+ auto CheckPotentialCalleeUse = [&](Function &PotentialCallee,
+ bool &UsedAssumedInformation) {
+ const auto *GIAA = A.getAAFor<AAGlobalValueInfo>(
+ *this, IRPosition::value(PotentialCallee), DepClassTy::OPTIONAL);
+ if (!GIAA || GIAA->isPotentialUse(CalleeUse))
+ return true;
+ UsedAssumedInformation = !GIAA->isAtFixpoint();
+ return false;
+ };
+
+ auto AddPotentialCallees = [&]() {
+ for (auto *PotentialCallee : PotentialCallees) {
+ bool UsedAssumedInformation = false;
+ if (CheckPotentialCalleeUse(*PotentialCallee, UsedAssumedInformation))
+ AssumedCalleesNow.insert(PotentialCallee);
+ }
+ };
+
+ // Use simplification to find potential callees, if !callees was present,
+ // fallback to that set if necessary.
+ bool UsedAssumedInformation = false;
+ SmallVector<AA::ValueAndContext> Values;
+ if (!A.getAssumedSimplifiedValues(IRPosition::value(*FP), this, Values,
+ AA::ValueScope::AnyScope,
+ UsedAssumedInformation)) {
+ if (PotentialCallees.empty())
+ return indicatePessimisticFixpoint();
+ AddPotentialCallees();
+ }
+
+ // Try to find a reason for \p Fn not to be a potential callee. If none was
+ // found, add it to the assumed callees set.
+ auto CheckPotentialCallee = [&](Function &Fn) {
+ if (!PotentialCallees.empty() && !PotentialCallees.count(&Fn))
+ return false;
+
+ auto &CachedResult = FilterResults[&Fn];
+ if (CachedResult.has_value())
+ return CachedResult.value();
+
+ bool UsedAssumedInformation = false;
+ if (!CheckPotentialCalleeUse(Fn, UsedAssumedInformation)) {
+ if (!UsedAssumedInformation)
+ CachedResult = false;
+ return false;
+ }
+
+ int NumFnArgs = Fn.arg_size();
+ int NumCBArgs = CB->arg_size();
+
+ // Check if any excess argument (which we fill up with poison) is known to
+ // be UB on undef.
+ for (int I = NumCBArgs; I < NumFnArgs; ++I) {
+ bool IsKnown = false;
+ if (AA::hasAssumedIRAttr<Attribute::NoUndef>(
+ A, this, IRPosition::argument(*Fn.getArg(I)),
+ DepClassTy::OPTIONAL, IsKnown)) {
+ if (IsKnown)
+ CachedResult = false;
+ return false;
+ }
+ }
+
+ CachedResult = true;
+ return true;
+ };
+
+ // Check simplification result, prune known UB callees, also restrict it to
+ // the !callees set, if present.
+ for (auto &VAC : Values) {
+ if (isa<UndefValue>(VAC.getValue()))
+ continue;
+ if (isa<ConstantPointerNull>(VAC.getValue()) &&
+ VAC.getValue()->getType()->getPointerAddressSpace() == 0)
+ continue;
+ // TODO: Check for known UB, e.g., poison + noundef.
+ if (auto *VACFn = dyn_cast<Function>(VAC.getValue())) {
+ if (CheckPotentialCallee(*VACFn))
+ AssumedCalleesNow.insert(VACFn);
+ continue;
+ }
+ if (!PotentialCallees.empty()) {
+ AddPotentialCallees();
+ break;
+ }
+ AllCalleesKnownNow = false;
+ }
+
+ if (AssumedCalleesNow == AssumedCallees &&
+ AllCalleesKnown == AllCalleesKnownNow)
+ return ChangeStatus::UNCHANGED;
+
+ std::swap(AssumedCallees, AssumedCalleesNow);
+ AllCalleesKnown = AllCalleesKnownNow;
+ return ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ // If we can't specialize at all, give up now.
+ if (!AllCalleesKnown && AssumedCallees.empty())
+ return ChangeStatus::UNCHANGED;
+
+ CallBase *CB = cast<CallBase>(getCtxI());
+ bool UsedAssumedInformation = false;
+ if (A.isAssumedDead(*CB, this, /*LivenessAA=*/nullptr,
+ UsedAssumedInformation))
+ return ChangeStatus::UNCHANGED;
+
+ ChangeStatus Changed = ChangeStatus::UNCHANGED;
+ Value *FP = CB->getCalledOperand();
+ if (FP->getType()->getPointerAddressSpace())
+ FP = new AddrSpaceCastInst(FP, PointerType::get(FP->getType(), 0),
+ FP->getName() + ".as0", CB->getIterator());
+
+ bool CBIsVoid = CB->getType()->isVoidTy();
+ BasicBlock::iterator IP = CB->getIterator();
+ FunctionType *CSFT = CB->getFunctionType();
+ SmallVector<Value *> CSArgs(CB->arg_begin(), CB->arg_end());
+
+ // If we know all callees and there are none, the call site is (effectively)
+ // dead (or UB).
+ if (AssumedCallees.empty()) {
+ assert(AllCalleesKnown &&
+ "Expected all callees to be known if there are none.");
+ A.changeToUnreachableAfterManifest(CB);
+ return ChangeStatus::CHANGED;
+ }
+
+ // Special handling for the single callee case.
+ if (AllCalleesKnown && AssumedCallees.size() == 1) {
+ auto *NewCallee = AssumedCallees.front();
+ if (isLegalToPromote(*CB, NewCallee)) {
+ promoteCall(*CB, NewCallee, nullptr);
+ return ChangeStatus::CHANGED;
+ }
+ Instruction *NewCall =
+ CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,
+ CB->getName(), CB->getIterator());
+ if (!CBIsVoid)
+ A.changeAfterManifest(IRPosition::callsite_returned(*CB), *NewCall);
+ A.deleteAfterManifest(*CB);
+ return ChangeStatus::CHANGED;
+ }
+
+ // For each potential value we create a conditional
+ //
+ // ```
+ // if (ptr == value) value(args);
+ // else ...
+ // ```
+ //
+ bool SpecializedForAnyCallees = false;
+ bool SpecializedForAllCallees = AllCalleesKnown;
+ ICmpInst *LastCmp = nullptr;
+ SmallVector<Function *, 8> SkippedAssumedCallees;
+ SmallVector<std::pair<CallInst *, Instruction *>> NewCalls;
+ for (Function *NewCallee : AssumedCallees) {
+ if (!A.shouldSpecializeCallSiteForCallee(*this, *CB, *NewCallee)) {
+ SkippedAssumedCallees.push_back(NewCallee);
+ SpecializedForAllCallees = false;
+ continue;
+ }
+ SpecializedForAnyCallees = true;
+
+ LastCmp = new ICmpInst(IP, llvm::CmpInst::ICMP_EQ, FP, NewCallee);
+ Instruction *ThenTI =
+ SplitBlockAndInsertIfThen(LastCmp, IP, /* Unreachable */ false);
+ BasicBlock *CBBB = CB->getParent();
+ A.registerManifestAddedBasicBlock(*ThenTI->getParent());
+ A.registerManifestAddedBasicBlock(*IP->getParent());
+ auto *SplitTI = cast<BranchInst>(LastCmp->getNextNode());
+ BasicBlock *ElseBB;
+ if (&*IP == CB) {
+ ElseBB = BasicBlock::Create(ThenTI->getContext(), "",
+ ThenTI->getFunction(), CBBB);
+ A.registerManifestAddedBasicBlock(*ElseBB);
+ IP = BranchInst::Create(CBBB, ElseBB)->getIterator();
+ SplitTI->replaceUsesOfWith(CBBB, ElseBB);
+ } else {
+ ElseBB = IP->getParent();
+ ThenTI->replaceUsesOfWith(ElseBB, CBBB);
+ }
+ CastInst *RetBC = nullptr;
+ CallInst *NewCall = nullptr;
+ if (isLegalToPromote(*CB, NewCallee)) {
+ auto *CBClone = cast<CallBase>(CB->clone());
+ CBClone->insertBefore(ThenTI);
+ NewCall = &cast<CallInst>(promoteCall(*CBClone, NewCallee, &RetBC));
+ } else {
+ NewCall = CallInst::Create(FunctionCallee(CSFT, NewCallee), CSArgs,
+ CB->getName(), ThenTI->getIterator());
+ }
+ NewCalls.push_back({NewCall, RetBC});
+ }
+
+ auto AttachCalleeMetadata = [&](CallBase &IndirectCB) {
+ if (!AllCalleesKnown)
+ return ChangeStatus::UNCHANGED;
+ MDBuilder MDB(IndirectCB.getContext());
+ MDNode *Callees = MDB.createCallees(SkippedAssumedCallees);
+ IndirectCB.setMetadata(LLVMContext::MD_callees, Callees);
+ return ChangeStatus::CHANGED;
+ };
+
+ if (!SpecializedForAnyCallees)
+ return AttachCalleeMetadata(*CB);
+
+ // Check if we need the fallback indirect call still.
+ if (SpecializedForAllCallees) {
+ LastCmp->replaceAllUsesWith(ConstantInt::getTrue(LastCmp->getContext()));
+ LastCmp->eraseFromParent();
+ new UnreachableInst(IP->getContext(), IP);
+ IP->eraseFromParent();
+ } else {
+ auto *CBClone = cast<CallInst>(CB->clone());
+ CBClone->setName(CB->getName());
+ CBClone->insertBefore(*IP->getParent(), IP);
+ NewCalls.push_back({CBClone, nullptr});
+ AttachCalleeMetadata(*CBClone);
+ }
+
+ // Check if we need a PHI to merge the results.
+ if (!CBIsVoid) {
+ auto *PHI = PHINode::Create(CB->getType(), NewCalls.size(),
+ CB->getName() + ".phi",
+ CB->getParent()->getFirstInsertionPt());
+ for (auto &It : NewCalls) {
+ CallBase *NewCall = It.first;
+ Instruction *CallRet = It.second ? It.second : It.first;
+ if (CallRet->getType() == CB->getType())
+ PHI->addIncoming(CallRet, CallRet->getParent());
+ else if (NewCall->getType()->isVoidTy())
+ PHI->addIncoming(PoisonValue::get(CB->getType()),
+ NewCall->getParent());
+ else
+ llvm_unreachable("Call return should match or be void!");
+ }
+ A.changeAfterManifest(IRPosition::callsite_returned(*CB), *PHI);
+ }
+
+ A.deleteAfterManifest(*CB);
+ Changed = ChangeStatus::CHANGED;
+
+ return Changed;
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ return std::string(AllCalleesKnown ? "eliminate" : "specialize") +
+ " indirect call site with " + std::to_string(AssumedCallees.size()) +
+ " functions";
+ }
+
+ void trackStatistics() const override {
+ if (AllCalleesKnown) {
+ STATS_DECLTRACK(
+ Eliminated, CallSites,
+ "Number of indirect call sites eliminated via specialization")
+ } else {
+ STATS_DECLTRACK(Specialized, CallSites,
+ "Number of indirect call sites specialized")
+ }
+ }
+
+ bool foreachCallee(function_ref<bool(Function *)> CB) const override {
+ return isValidState() && AllCalleesKnown && all_of(AssumedCallees, CB);
+ }
+
+private:
+ /// Map to remember filter results.
+ DenseMap<Function *, std::optional<bool>> FilterResults;
+
+ /// If the !callee metadata was present, this set will contain all potential
+ /// callees (superset).
+ SmallSetVector<Function *, 4> PotentialCallees;
+
+ /// This set contains all currently assumed calllees, which might grow over
+ /// time.
+ SmallSetVector<Function *, 4> AssumedCallees;
+
+ /// Flag to indicate if all possible callees are in the AssumedCallees set or
+ /// if there could be others.
+ bool AllCalleesKnown = true;
+};
+} // namespace
+
+/// ------------------------ Address Space ------------------------------------
+namespace {
+struct AAAddressSpaceImpl : public AAAddressSpace {
+ AAAddressSpaceImpl(const IRPosition &IRP, Attributor &A)
+ : AAAddressSpace(IRP, A) {}
+
+ int32_t getAddressSpace() const override {
+ assert(isValidState() && "the AA is invalid");
+ return AssumedAddressSpace;
+ }
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ assert(getAssociatedType()->isPtrOrPtrVectorTy() &&
+ "Associated value is not a pointer");
+ }
+
+ ChangeStatus updateImpl(Attributor &A) override {
+ int32_t OldAddressSpace = AssumedAddressSpace;
+ auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(getIRPosition(), this,
+ DepClassTy::REQUIRED);
+ auto Pred = [&](Value &Obj) {
+ if (isa<UndefValue>(&Obj))
+ return true;
+ return takeAddressSpace(Obj.getType()->getPointerAddressSpace());
+ };
+
+ if (!AUO->forallUnderlyingObjects(Pred))
+ return indicatePessimisticFixpoint();
+
+ return OldAddressSpace == AssumedAddressSpace ? ChangeStatus::UNCHANGED
+ : ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+ Value *AssociatedValue = &getAssociatedValue();
+ Value *OriginalValue = peelAddrspacecast(AssociatedValue);
+ if (getAddressSpace() == NoAddressSpace ||
+ static_cast<uint32_t>(getAddressSpace()) ==
+ getAssociatedType()->getPointerAddressSpace())
+ return ChangeStatus::UNCHANGED;
+
+ Type *NewPtrTy = PointerType::get(getAssociatedType()->getContext(),
+ static_cast<uint32_t>(getAddressSpace()));
+ bool UseOriginalValue =
+ OriginalValue->getType()->getPointerAddressSpace() ==
+ static_cast<uint32_t>(getAddressSpace());
+
+ bool Changed = false;
+
+ auto MakeChange = [&](Instruction *I, Use &U) {
+ Changed = true;
+ if (UseOriginalValue) {
+ A.changeUseAfterManifest(U, *OriginalValue);
+ return;
+ }
+ Instruction *CastInst = new AddrSpaceCastInst(OriginalValue, NewPtrTy);
+ CastInst->insertBefore(cast<Instruction>(I));
+ A.changeUseAfterManifest(U, *CastInst);
+ };
+
+ auto Pred = [&](const Use &U, bool &) {
+ if (U.get() != AssociatedValue)
+ return true;
+ auto *Inst = dyn_cast<Instruction>(U.getUser());
+ if (!Inst)
+ return true;
+ // This is a WA to make sure we only change uses from the corresponding
+ // CGSCC if the AA is run on CGSCC instead of the entire module.
+ if (!A.isRunOn(Inst->getFunction()))
+ return true;
+ if (isa<LoadInst>(Inst))
+ MakeChange(Inst, const_cast<Use &>(U));
+ if (isa<StoreInst>(Inst)) {
+ // We only make changes if the use is the pointer operand.
+ if (U.getOperandNo() == 1)
+ MakeChange(Inst, const_cast<Use &>(U));
+ }
+ return true;
+ };
+
+ // It doesn't matter if we can't check all uses as we can simply
+ // conservatively ignore those that can not be visited.
+ (void)A.checkForAllUses(Pred, *this, getAssociatedValue(),
+ /* CheckBBLivenessOnly */ true);
+
+ return Changed ? ChangeStatus::CHANGED : ChangeStatus::UNCHANGED;
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ if (!isValidState())
+ return "addrspace(<invalid>)";
+ return "addrspace(" +
+ (AssumedAddressSpace == NoAddressSpace
+ ? "none"
+ : std::to_string(AssumedAddressSpace)) +
+ ")";
+ }
+
+private:
+ int32_t AssumedAddressSpace = NoAddressSpace;
+
+ bool takeAddressSpace(int32_t AS) {
+ if (AssumedAddressSpace == NoAddressSpace) {
+ AssumedAddressSpace = AS;
+ return true;
+ }
+ return AssumedAddressSpace == AS;
+ }
+
+ static Value *peelAddrspacecast(Value *V) {
+ if (auto *I = dyn_cast<AddrSpaceCastInst>(V))
+ return peelAddrspacecast(I->getPointerOperand());
+ if (auto *C = dyn_cast<ConstantExpr>(V))
+ if (C->getOpcode() == Instruction::AddrSpaceCast)
+ return peelAddrspacecast(C->getOperand(0));
+ return V;
+ }
+};
+
+struct AAAddressSpaceFloating final : AAAddressSpaceImpl {
+ AAAddressSpaceFloating(const IRPosition &IRP, Attributor &A)
+ : AAAddressSpaceImpl(IRP, A) {}
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(addrspace);
+ }
+};
+
+struct AAAddressSpaceReturned final : AAAddressSpaceImpl {
+ AAAddressSpaceReturned(const IRPosition &IRP, Attributor &A)
+ : AAAddressSpaceImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ // TODO: we don't rewrite function argument for now because it will need to
+ // rewrite the function signature and all call sites.
+ (void)indicatePessimisticFixpoint();
+ }
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FNRET_ATTR(addrspace);
+ }
+};
+
+struct AAAddressSpaceCallSiteReturned final : AAAddressSpaceImpl {
+ AAAddressSpaceCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AAAddressSpaceImpl(IRP, A) {}
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSRET_ATTR(addrspace);
+ }
+};
+
+struct AAAddressSpaceArgument final : AAAddressSpaceImpl {
+ AAAddressSpaceArgument(const IRPosition &IRP, Attributor &A)
+ : AAAddressSpaceImpl(IRP, A) {}
+
+ void trackStatistics() const override { STATS_DECLTRACK_ARG_ATTR(addrspace); }
+};
+
+struct AAAddressSpaceCallSiteArgument final : AAAddressSpaceImpl {
+ AAAddressSpaceCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAAddressSpaceImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ // TODO: we don't rewrite call site argument for now because it will need to
+ // rewrite the function signature of the callee.
+ (void)indicatePessimisticFixpoint();
+ }
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSARG_ATTR(addrspace);
+ }
+};
+} // namespace
+
+/// ----------- Allocation Info ----------
+namespace {
+struct AAAllocationInfoImpl : public AAAllocationInfo {
+ AAAllocationInfoImpl(const IRPosition &IRP, Attributor &A)
+ : AAAllocationInfo(IRP, A) {}
+
+ std::optional<TypeSize> getAllocatedSize() const override {
+ assert(isValidState() && "the AA is invalid");
+ return AssumedAllocatedSize;
+ }
+
+ const NewOffsetsTy &getNewOffsets() const override {
+ assert(isValidState() && "the AA is invalid");
+ return NewComputedOffsets;
+ }
+
+ std::optional<TypeSize> findInitialAllocationSize(Instruction *I,
+ const DataLayout &DL) {
+
+ // TODO: implement case for malloc like instructions
+ switch (I->getOpcode()) {
+ case Instruction::Alloca: {
+ AllocaInst *AI = cast<AllocaInst>(I);
+ return AI->getAllocationSize(DL);
+ }
+ default:
+ return std::nullopt;
+ }
+ }
+
+ ChangeStatus updateImpl(Attributor &A) override {
+
+ const IRPosition &IRP = getIRPosition();
+ Instruction *I = IRP.getCtxI();
+
+ // TODO: update check for malloc like calls
+ if (!isa<AllocaInst>(I))
+ return indicatePessimisticFixpoint();
+
+ bool IsKnownNoCapture;
+ if (!AA::hasAssumedIRAttr<Attribute::NoCapture>(
+ A, this, IRP, DepClassTy::OPTIONAL, IsKnownNoCapture))
+ return indicatePessimisticFixpoint();
+
+ const AAPointerInfo *PI =
+ A.getOrCreateAAFor<AAPointerInfo>(IRP, *this, DepClassTy::REQUIRED);
+
+ if (!PI)
+ return indicatePessimisticFixpoint();
+
+ if (!PI->getState().isValidState())
+ return indicatePessimisticFixpoint();
+
+ const DataLayout &DL = A.getDataLayout();
+ const auto AllocationSize = findInitialAllocationSize(I, DL);
+
+ // If allocation size is nullopt, we give up.
+ if (!AllocationSize)
+ return indicatePessimisticFixpoint();
+
+ // For zero sized allocations, we give up.
+ // Since we can't reduce further
+ if (*AllocationSize == 0)
+ return indicatePessimisticFixpoint();
+
+ int64_t NumBins = PI->numOffsetBins();
+
+ if (NumBins == 0) {
+ auto NewAllocationSize = std::optional<TypeSize>(TypeSize(0, false));
+ if (!changeAllocationSize(NewAllocationSize))
+ return ChangeStatus::UNCHANGED;
+ return ChangeStatus::CHANGED;
+ }
+
+ // For each access bin
+ // Compute its new start Offset and store the results in a new map
+ // (NewOffsetBins).
+ int64_t PrevBinEndOffset = 0;
+ bool ChangedOffsets = false;
+ for (AAPointerInfo::OffsetBinsTy::const_iterator It = PI->begin();
+ It != PI->end(); It++) {
+ const AA::RangeTy &OldRange = It->getFirst();
+ int64_t NewStartOffset = PrevBinEndOffset;
+ int64_t NewEndOffset = NewStartOffset + OldRange.Size;
+ PrevBinEndOffset = NewEndOffset;
+
+ ChangedOffsets |= setNewOffsets(OldRange, OldRange.Offset, NewStartOffset,
+ OldRange.Size);
+ }
+
+ // Set the new size of the allocation, the new size of the Allocation should
+ // be the size of NewEndOffset * 8, in bits.
+ auto NewAllocationSize =
+ std::optional<TypeSize>(TypeSize(PrevBinEndOffset * 8, false));
+
+ if (!changeAllocationSize(NewAllocationSize))
+ return ChangeStatus::UNCHANGED;
+
+ if (!ChangedOffsets)
+ return ChangeStatus::UNCHANGED;
+
+ return ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractAttribute::manifest(...).
+ ChangeStatus manifest(Attributor &A) override {
+
+ assert(isValidState() &&
+ "Manifest should only be called if the state is valid.");
+
+ bool Changed = false;
+ const IRPosition &IRP = getIRPosition();
+ Instruction *I = IRP.getCtxI();
+
+ auto FixedAllocatedSizeInBits = getAllocatedSize()->getFixedValue();
+
+ unsigned long NumBytesToAllocate = (FixedAllocatedSizeInBits + 7) / 8;
+ switch (I->getOpcode()) {
+ // TODO: add case for malloc like calls
+ case Instruction::Alloca: {
+
+ AllocaInst *AI = cast<AllocaInst>(I);
+
+ Type *CharType = Type::getInt8Ty(I->getContext());
+
+ Type *CharArrayType = ArrayType::get(CharType, NumBytesToAllocate);
+
+ BasicBlock::iterator insertPt = AI->getIterator();
+ insertPt = std::next(insertPt);
+ AllocaInst *NewAllocaInst = new AllocaInst(
+ CharArrayType, AI->getAddressSpace(), AI->getName(), insertPt);
+
+ Changed |= A.changeAfterManifest(IRPosition::inst(*AI), *NewAllocaInst);
+ break;
+ }
+ default:
+ break;
+ }
+
+ const AAPointerInfo *PI =
+ A.getOrCreateAAFor<AAPointerInfo>(IRP, *this, DepClassTy::REQUIRED);
+
+ if (!PI)
+ return ChangeStatus::UNCHANGED;
+
+ if (!PI->getState().isValidState())
+ return ChangeStatus::UNCHANGED;
+
+ const auto &NewOffsetsMap = getNewOffsets();
+ for (AAPointerInfo::OffsetBinsTy::const_iterator It = PI->begin();
+ It != PI->end(); It++) {
+
+ const auto &OldOffsetRange = It->getFirst();
+
+ // If the OldOffsetRange is not in the map, offsets for that bin did not
+ // change We should just continue and skip changing the offsets in that
+ // case
+ if (!NewOffsetsMap.contains(OldOffsetRange))
+ continue;
+
+ const auto &NewOffsetRange = NewOffsetsMap.lookup(OldOffsetRange);
+ int64_t ShiftValue = OldOffsetRange.Offset - NewOffsetRange.Offset;
+ for (const auto AccIndex : It->getSecond()) {
+
+ const auto &AccessInstruction = PI->getBinAccess(AccIndex);
+ auto *LocalInst = AccessInstruction.getLocalInst();
+
+ switch (LocalInst->getOpcode()) {
+ case Instruction::Load: {
+ LoadInst *OldLoadInst = cast<LoadInst>(LocalInst);
+ Value *PointerOperand = OldLoadInst->getPointerOperand();
+
+ // We need to shift the old start offset by the difference between the
+ // Old offset and the New offset.
+ Value *ShiftValueForPointerOperand = ConstantInt::get(
+ OldLoadInst->getContext(),
+ APInt(32, ShiftValue));
+ // Cast ptr to 32 bit integer
+ CastInst *PtrOperandToInt = PtrToIntInst::CreatePointerCast(
+ PointerOperand, Type::getInt32Ty(OldLoadInst->getContext()), "",
+ OldLoadInst);
+ // Subtract shift amount from old offset
+ BinaryOperator *NewPointerOperandInt = BinaryOperator::Create(
+ Instruction::Sub, PtrOperandToInt, ShiftValueForPointerOperand,
+ "", OldLoadInst);
+ // Convert Int to ptr.
+ CastInst *IntToPtr = new IntToPtrInst(
+ NewPointerOperandInt, OldLoadInst->getPointerOperandType(), "",
+ OldLoadInst);
+ LoadInst *NewLoadInst =
+ new LoadInst(OldLoadInst->getType(), IntToPtr,
+ OldLoadInst->getName(), OldLoadInst);
+ Changed |= A.changeAfterManifest(IRPosition::inst(*OldLoadInst),
+ *NewLoadInst);
+ break;
+ }
+ case Instruction::Store: {
+ StoreInst *OldStoreInst = cast<StoreInst>(LocalInst);
+ Value *PointerOperand = OldStoreInst->getPointerOperand();
+
+ Value *ShiftValueForPointerOperand = ConstantInt::get(
+ OldStoreInst->getContext(),
+ APInt(32, ShiftValue));
+
+ // Cast ptr to 32 bit integer
+ CastInst *PtrOperandToInt = PtrToIntInst::CreatePointerCast(
+ PointerOperand, Type::getInt32Ty(OldStoreInst->getContext()), "",
+ OldStoreInst);
+ // Subtract shift amount from old offset
+ BinaryOperator *NewPointerOperandInt = BinaryOperator::Create(
+ Instruction::Sub, PtrOperandToInt, ShiftValueForPointerOperand,
+ "", OldStoreInst);
+ // Convert Int to ptr.
+ CastInst *IntToPtr = new IntToPtrInst(
+ NewPointerOperandInt, OldStoreInst->getPointerOperandType(), "",
+ OldStoreInst);
+ StoreInst *NewStoreInst = new StoreInst(
+ OldStoreInst->getValueOperand(), IntToPtr, OldStoreInst);
+ Changed |= A.changeAfterManifest(IRPosition::inst(*OldStoreInst),
+ *NewStoreInst);
+ break;
+ }
+ }
+ }
+ }
+
+ if (!Changed)
+ return ChangeStatus::UNCHANGED;
+ return ChangeStatus::CHANGED;
+ }
+
+ /// See AbstractAttribute::getAsStr().
+ const std::string getAsStr(Attributor *A) const override {
+ if (!isValidState())
+ return "allocationinfo(<invalid>)";
+ return "allocationinfo(" +
+ (AssumedAllocatedSize == HasNoAllocationSize
+ ? "none"
+ : std::to_string(AssumedAllocatedSize->getFixedValue())) +
+ ")";
+ }
+
+ void dumpNewOffsetBins(raw_ostream &O) {
+
+ O << "Printing Map from [OldOffsetsRange] : [NewOffsetsRange] if the "
+ "offsets changed."
+ << "\n";
+ const auto &NewOffsetsMap = getNewOffsets();
+ for (auto It = NewOffsetsMap.begin(); It != NewOffsetsMap.end(); It++) {
+
+ const auto &OldRange = It->getFirst();
+ const auto &NewRange = It->getSecond();
+
+ O << "[" << OldRange.Offset << "," << OldRange.Offset + OldRange.Size
+ << "] : ";
+ O << "[" << NewRange.Offset << "," << NewRange.Offset + NewRange.Size
+ << "]";
+ O << "\n";
+ }
+ }
+
+private:
+ std::optional<TypeSize> AssumedAllocatedSize = HasNoAllocationSize;
+ NewOffsetsTy NewComputedOffsets;
+
+ // Maintain the computed allocation size of the object.
+ // Returns (bool) weather the size of the allocation was modified or not.
+ bool changeAllocationSize(std::optional<TypeSize> Size) {
+ if (AssumedAllocatedSize == HasNoAllocationSize ||
+ AssumedAllocatedSize != Size) {
+ AssumedAllocatedSize = Size;
+ return true;
+ }
+ return false;
+ }
+
+ // Maps an old byte range to its new Offset range in the new allocation.
+ // Returns (bool) weather the old byte range's offsets changed or not.
+ bool setNewOffsets(const AA::RangeTy &OldRange, int64_t OldOffset,
+ int64_t NewComputedOffset, int64_t Size) {
+
+ if (OldOffset == NewComputedOffset)
+ return false;
+
+ AA::RangeTy &NewRange = NewComputedOffsets.getOrInsertDefault(OldRange);
+ NewRange.Offset = NewComputedOffset;
+ NewRange.Size = Size;
+
+ return true;
+ }
+};
+
+struct AAAllocationInfoFloating : AAAllocationInfoImpl {
+ AAAllocationInfoFloating(const IRPosition &IRP, Attributor &A)
+ : AAAllocationInfoImpl(IRP, A) {}
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FLOATING_ATTR(allocationinfo);
+ }
+};
+
+struct AAAllocationInfoReturned : AAAllocationInfoImpl {
+ AAAllocationInfoReturned(const IRPosition &IRP, Attributor &A)
+ : AAAllocationInfoImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+ // TODO: we don't rewrite function argument for now because it will need to
+ // rewrite the function signature and all call sites
+ (void)indicatePessimisticFixpoint();
+ }
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_FNRET_ATTR(allocationinfo);
+ }
+};
+
+struct AAAllocationInfoCallSiteReturned : AAAllocationInfoImpl {
+ AAAllocationInfoCallSiteReturned(const IRPosition &IRP, Attributor &A)
+ : AAAllocationInfoImpl(IRP, A) {}
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSRET_ATTR(allocationinfo);
+ }
+};
+
+struct AAAllocationInfoArgument : AAAllocationInfoImpl {
+ AAAllocationInfoArgument(const IRPosition &IRP, Attributor &A)
+ : AAAllocationInfoImpl(IRP, A) {}
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_ARG_ATTR(allocationinfo);
+ }
+};
+
+struct AAAllocationInfoCallSiteArgument : AAAllocationInfoImpl {
+ AAAllocationInfoCallSiteArgument(const IRPosition &IRP, Attributor &A)
+ : AAAllocationInfoImpl(IRP, A) {}
+
+ /// See AbstractAttribute::initialize(...).
+ void initialize(Attributor &A) override {
+
+ (void)indicatePessimisticFixpoint();
+ }
+
+ void trackStatistics() const override {
+ STATS_DECLTRACK_CSARG_ATTR(allocationinfo);
+ }
+};
+} // namespace
+
+const char AANoUnwind::ID = 0;
+const char AANoSync::ID = 0;
+const char AANoFree::ID = 0;
+const char AANonNull::ID = 0;
+const char AAMustProgress::ID = 0;
+const char AANoRecurse::ID = 0;
+const char AANonConvergent::ID = 0;
+const char AAWillReturn::ID = 0;
+const char AAUndefinedBehavior::ID = 0;
+const char AANoAlias::ID = 0;
+const char AAIntraFnReachability::ID = 0;
+const char AANoReturn::ID = 0;
+const char AAIsDead::ID = 0;
+const char AADereferenceable::ID = 0;
+const char AAAlign::ID = 0;
+const char AAInstanceInfo::ID = 0;
+const char AANoCapture::ID = 0;
+const char AAValueSimplify::ID = 0;
+const char AAHeapToStack::ID = 0;
+const char AAPrivatizablePtr::ID = 0;
+const char AAMemoryBehavior::ID = 0;
+const char AAMemoryLocation::ID = 0;
+const char AAValueConstantRange::ID = 0;
+const char AAPotentialConstantValues::ID = 0;
+const char AAPotentialValues::ID = 0;
+const char AANoUndef::ID = 0;
+const char AANoFPClass::ID = 0;
+const char AACallEdges::ID = 0;
+const char AAInterFnReachability::ID = 0;
+const char AAPointerInfo::ID = 0;
+const char AAAssumptionInfo::ID = 0;
+const char AAUnderlyingObjects::ID = 0;
+const char AAAddressSpace::ID = 0;
+const char AAAllocationInfo::ID = 0;
+const char AAIndirectCallInfo::ID = 0;
+const char AAGlobalValueInfo::ID = 0;
+const char AADenormalFPMath::ID = 0;
+
+// Macro magic to create the static generator function for attributes that
+// follow the naming scheme.
+
+#define SWITCH_PK_INV(CLASS, PK, POS_NAME) \
+ case IRPosition::PK: \
+ llvm_unreachable("Cannot create " #CLASS " for a " POS_NAME " position!");
+
+#define SWITCH_PK_CREATE(CLASS, IRP, PK, SUFFIX) \
+ case IRPosition::PK: \
+ AA = new (A.Allocator) CLASS##SUFFIX(IRP, A); \
+ ++NumAAs; \
+ break;
+
+#define CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
+ CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
+ CLASS *AA = nullptr; \
+ switch (IRP.getPositionKind()) { \
+ SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
+ SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
+ SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
+ SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
+ SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
+ SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
+ } \
+ return *AA; \
+ }
+
+#define CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
+ CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
+ CLASS *AA = nullptr; \
+ switch (IRP.getPositionKind()) { \
+ SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
+ SWITCH_PK_INV(CLASS, IRP_FUNCTION, "function") \
+ SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
+ } \
+ return *AA; \
+ }
+
+#define CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(POS, SUFFIX, CLASS) \
+ CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
+ CLASS *AA = nullptr; \
+ switch (IRP.getPositionKind()) { \
+ SWITCH_PK_CREATE(CLASS, IRP, POS, SUFFIX) \
+ default: \
+ llvm_unreachable("Cannot create " #CLASS " for position otherthan " #POS \
+ " position!"); \
+ } \
+ return *AA; \
+ }
+
+#define CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
+ CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
+ CLASS *AA = nullptr; \
+ switch (IRP.getPositionKind()) { \
+ SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_RETURNED, Returned) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
+ } \
+ return *AA; \
+ }
+
+#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
+ CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
+ CLASS *AA = nullptr; \
+ switch (IRP.getPositionKind()) { \
+ SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
+ SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument") \
+ SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating") \
+ SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
+ SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned") \
+ SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument") \
+ SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site") \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
+ } \
+ return *AA; \
+ }
+
+#define CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS) \
+ CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) { \
+ CLASS *AA = nullptr; \
+ switch (IRP.getPositionKind()) { \
+ SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid") \
+ SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned") \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE, CallSite) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_FLOAT, Floating) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_ARGUMENT, Argument) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_RETURNED, CallSiteReturned) \
+ SWITCH_PK_CREATE(CLASS, IRP, IRP_CALL_SITE_ARGUMENT, CallSiteArgument) \
+ } \
+ return *AA; \
+ }
+
+CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)
+CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)
+CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoRecurse)
+CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAWillReturn)
+CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoReturn)
+CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryLocation)
+CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AACallEdges)
+CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAssumptionInfo)
+CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMustProgress)
+
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonNull)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoAlias)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPrivatizablePtr)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADereferenceable)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAlign)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInstanceInfo)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueConstantRange)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialConstantValues)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
+CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
+
+CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
+CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIsDead)
+CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)
+CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUnderlyingObjects)
+
+CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_CALL_SITE, CallSite,
+ AAIndirectCallInfo)
+CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION(IRP_FLOAT, Floating,
+ AAGlobalValueInfo)
+
+CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)
+CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAUndefinedBehavior)
+CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANonConvergent)
+CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAIntraFnReachability)
+CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInterFnReachability)
+CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AADenormalFPMath)
+
+CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAMemoryBehavior)
+
+#undef CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION
+#undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
+#undef CREATE_NON_RET_ABSTRACT_ATTRIBUTE_FOR_POSITION
+#undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
+#undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
+#undef CREATE_ABSTRACT_ATTRIBUTE_FOR_ONE_POSITION
+#undef SWITCH_PK_CREATE
+#undef SWITCH_PK_INV
More information about the llvm-commits
mailing list