[llvm] c117d77 - [ConstantFold] Refactor load folding
Nikita Popov via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 5 09:08:10 PDT 2021
Author: Nikita Popov
Date: 2021-10-05T18:07:57+02:00
New Revision: c117d77e937f410a2b9c54eaec0fc5fe2a653399
URL: https://github.com/llvm/llvm-project/commit/c117d77e937f410a2b9c54eaec0fc5fe2a653399
DIFF: https://github.com/llvm/llvm-project/commit/c117d77e937f410a2b9c54eaec0fc5fe2a653399.diff
LOG: [ConstantFold] Refactor load folding
This refactors load folding to happen in two cleanly separated
steps: ConstantFoldLoadFromConstPtr() takes a pointer to load from
and decomposes it into a constant initializer base and an offset.
Then ConstantFoldLoadFromConst() loads from that initializer at
the given offset. This makes the core logic independent of having
actual GEP expressions (and those GEP expressions having certain
structure) and will allow exposing ConstantFoldLoadFromConst() as
an independent API in the future.
This is mostly only a refactoring, but it does make the folding
logic slightly more powerful.
Differential Revision: https://reviews.llvm.org/D111023
Added:
Modified:
llvm/lib/Analysis/ConstantFolding.cpp
llvm/test/Transforms/InstSimplify/ConstProp/loads.ll
llvm/test/Transforms/InstSimplify/load.ll
Removed:
################################################################################
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 9336361c958b0..98d886c894fc7 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -63,11 +63,6 @@
using namespace llvm;
namespace {
-Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
- ArrayRef<Constant *> Ops,
- const DataLayout &DL,
- const TargetLibraryInfo *TLI,
- bool ForLoadOperand);
//===----------------------------------------------------------------------===//
// Constant Folding internal helper functions
@@ -550,19 +545,16 @@ bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset, unsigned char *CurPtr,
return false;
}
-Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
- const DataLayout &DL) {
+Constant *FoldReinterpretLoadFromConst(Constant *C, Type *LoadTy,
+ int64_t Offset, const DataLayout &DL) {
// Bail out early. Not expect to load from scalable global variable.
if (isa<ScalableVectorType>(LoadTy))
return nullptr;
- auto *PTy = cast<PointerType>(C->getType());
auto *IntType = dyn_cast<IntegerType>(LoadTy);
// If this isn't an integer load we can't fold it directly.
if (!IntType) {
- unsigned AS = PTy->getAddressSpace();
-
// If this is a float/double load, we can try folding it as an int32/64 load
// and then bitcast the result. This can be useful for union cases. Note
// that address spaces don't matter here since we're not going to result in
@@ -580,8 +572,7 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
} else
return nullptr;
- C = FoldBitCast(C, MapTy->getPointerTo(AS), DL);
- if (Constant *Res = FoldReinterpretLoadFromConstPtr(C, MapTy, DL)) {
+ if (Constant *Res = FoldReinterpretLoadFromConst(C, MapTy, Offset, DL)) {
if (Res->isNullValue() && !LoadTy->isX86_MMXTy() &&
!LoadTy->isX86_AMXTy())
// Materializing a zero can be done trivially without a bitcast
@@ -607,19 +598,7 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
if (BytesLoaded > 32 || BytesLoaded == 0)
return nullptr;
- GlobalValue *GVal;
- APInt OffsetAI;
- if (!IsConstantOffsetFromGlobal(C, GVal, OffsetAI, DL))
- return nullptr;
-
- auto *GV = dyn_cast<GlobalVariable>(GVal);
- if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() ||
- !GV->getInitializer()->getType()->isSized())
- return nullptr;
-
- int64_t Offset = OffsetAI.getSExtValue();
- int64_t InitializerSize =
- DL.getTypeAllocSize(GV->getInitializer()->getType()).getFixedSize();
+ int64_t InitializerSize = DL.getTypeAllocSize(C->getType()).getFixedSize();
// If we're not accessing anything in this constant, the result is undefined.
if (Offset <= -1 * static_cast<int64_t>(BytesLoaded))
@@ -640,7 +619,7 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
Offset = 0;
}
- if (!ReadDataFromGlobal(GV->getInitializer(), Offset, CurPtr, BytesLeft, DL))
+ if (!ReadDataFromGlobal(C, Offset, CurPtr, BytesLeft, DL))
return nullptr;
APInt ResultVal = APInt(IntType->getBitWidth(), 0);
@@ -661,111 +640,64 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
return ConstantInt::get(IntType->getContext(), ResultVal);
}
-Constant *ConstantFoldLoadThroughBitcastExpr(ConstantExpr *CE, Type *DestTy,
- const DataLayout &DL) {
- auto *SrcPtr = CE->getOperand(0);
- if (!SrcPtr->getType()->isPointerTy())
+/// If this Offset points exactly to the start of an aggregate element, return
+/// that element, otherwise return nullptr.
+Constant *getConstantAtOffset(Constant *Base, APInt Offset,
+ const DataLayout &DL) {
+ if (Offset.isZero())
+ return Base;
+
+ if (!isa<ConstantAggregate>(Base) && !isa<ConstantDataSequential>(Base))
return nullptr;
- return ConstantFoldLoadFromConstPtr(SrcPtr, DestTy, DL);
+ Type *ElemTy = Base->getType();
+ SmallVector<APInt> Indices = DL.getGEPIndicesForOffset(ElemTy, Offset);
+ if (!Offset.isZero() || !Indices[0].isZero())
+ return nullptr;
+
+ Constant *C = Base;
+ for (const APInt &Index : drop_begin(Indices)) {
+ if (Index.isNegative() || Index.getActiveBits() >= 32)
+ return nullptr;
+
+ C = C->getAggregateElement(Index.getZExtValue());
+ if (!C)
+ return nullptr;
+ }
+
+ return C;
+}
+
+Constant *ConstantFoldLoadFromConst(Constant *C, Type *Ty, const APInt &Offset,
+ const DataLayout &DL) {
+ if (Constant *AtOffset = getConstantAtOffset(C, Offset, DL))
+ if (Constant *Result = ConstantFoldLoadThroughBitcast(AtOffset, Ty, DL))
+ return Result;
+
+ // Try hard to fold loads from bitcasted strange and non-type-safe things.
+ if (Offset.getMinSignedBits() <= 64)
+ return FoldReinterpretLoadFromConst(C, Ty, Offset.getSExtValue(), DL);
+
+ return nullptr;
}
} // end anonymous namespace
Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
const DataLayout &DL) {
- // First, try the easy cases:
+ APInt Offset(DL.getIndexTypeSizeInBits(C->getType()), 0);
+ C = cast<Constant>(C->stripAndAccumulateConstantOffsets(
+ DL, Offset, /* AllowNonInbounds */ true));
+
if (auto *GV = dyn_cast<GlobalVariable>(C))
if (GV->isConstant() && GV->hasDefinitiveInitializer())
- return ConstantFoldLoadThroughBitcast(GV->getInitializer(), Ty, DL);
-
- if (auto *GA = dyn_cast<GlobalAlias>(C))
- if (GA->getAliasee() && !GA->isInterposable())
- return ConstantFoldLoadFromConstPtr(GA->getAliasee(), Ty, DL);
-
- // If the loaded value isn't a constant expr, we can't handle it.
- auto *CE = dyn_cast<ConstantExpr>(C);
- if (!CE)
- return nullptr;
-
- if (CE->getOpcode() == Instruction::GetElementPtr) {
- if (auto *GV = dyn_cast<GlobalVariable>(CE->getOperand(0))) {
- if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
- if (Constant *V = ConstantFoldLoadThroughGEPConstantExpr(
- GV->getInitializer(), CE, Ty, DL))
- return V;
- }
- } else {
- // Try to simplify GEP if the pointer operand wasn't a GlobalVariable.
- // SymbolicallyEvaluateGEP() with `ForLoadOperand = true` can potentially
- // simplify the GEP more than it normally would have been, but should only
- // be used for const folding loads.
- SmallVector<Constant *> Ops;
- for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I)
- Ops.push_back(cast<Constant>(CE->getOperand(I)));
- if (auto *Simplified = dyn_cast_or_null<ConstantExpr>(
- SymbolicallyEvaluateGEP(cast<GEPOperator>(CE), Ops, DL, nullptr,
- /*ForLoadOperand*/ true))) {
- // If the symbolically evaluated GEP is another GEP, we can only const
- // fold it if the resulting pointer operand is a GlobalValue. Otherwise
- // there is nothing else to simplify since the GEP is already in the
- // most simplified form.
- if (isa<GEPOperator>(Simplified)) {
- if (auto *GV = dyn_cast<GlobalVariable>(Simplified->getOperand(0))) {
- if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
- if (Constant *V = ConstantFoldLoadThroughGEPConstantExpr(
- GV->getInitializer(), Simplified, Ty, DL))
- return V;
- }
- }
- } else {
- return ConstantFoldLoadFromConstPtr(Simplified, Ty, DL);
- }
- }
- }
- }
-
- if (CE->getOpcode() == Instruction::BitCast)
- if (Constant *LoadedC = ConstantFoldLoadThroughBitcastExpr(CE, Ty, DL))
- return LoadedC;
-
- // Instead of loading constant c string, use corresponding integer value
- // directly if string length is small enough.
- StringRef Str;
- if (getConstantStringInfo(CE, Str) && !Str.empty()) {
- size_t StrLen = Str.size();
- unsigned NumBits = Ty->getPrimitiveSizeInBits();
- // Replace load with immediate integer if the result is an integer or fp
- // value.
- if ((NumBits >> 3) == StrLen + 1 && (NumBits & 7) == 0 &&
- (isa<IntegerType>(Ty) || Ty->isFloatingPointTy())) {
- APInt StrVal(NumBits, 0);
- APInt SingleChar(NumBits, 0);
- if (DL.isLittleEndian()) {
- for (unsigned char C : reverse(Str.bytes())) {
- SingleChar = static_cast<uint64_t>(C);
- StrVal = (StrVal << 8) | SingleChar;
- }
- } else {
- for (unsigned char C : Str.bytes()) {
- SingleChar = static_cast<uint64_t>(C);
- StrVal = (StrVal << 8) | SingleChar;
- }
- // Append NULL at the end.
- SingleChar = 0;
- StrVal = (StrVal << 8) | SingleChar;
- }
-
- Constant *Res = ConstantInt::get(CE->getContext(), StrVal);
- if (Ty->isFloatingPointTy())
- Res = ConstantExpr::getBitCast(Res, Ty);
- return Res;
- }
- }
+ if (Constant *Result = ConstantFoldLoadFromConst(GV->getInitializer(), Ty,
+ Offset, DL))
+ return Result;
// If this load comes from anywhere in a constant global, and if the global
// is all undef or zero, we know what it loads.
- if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(CE))) {
+ if (auto *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(C))) {
if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
if (GV->getInitializer()->isNullValue())
return Constant::getNullValue(Ty);
@@ -774,8 +706,7 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty,
}
}
- // Try hard to fold loads from bitcasted strange and non-type-safe things.
- return FoldReinterpretLoadFromConstPtr(CE, Ty, DL);
+ return nullptr;
}
namespace {
@@ -867,17 +798,10 @@ Constant *CastGEPIndices(Type *SrcElemTy, ArrayRef<Constant *> Ops,
}
/// Strip the pointer casts, but preserve the address space information.
-Constant *StripPtrCastKeepAS(Constant *Ptr, bool ForLoadOperand) {
+Constant *StripPtrCastKeepAS(Constant *Ptr) {
assert(Ptr->getType()->isPointerTy() && "Not a pointer type");
auto *OldPtrTy = cast<PointerType>(Ptr->getType());
Ptr = cast<Constant>(Ptr->stripPointerCasts());
- if (ForLoadOperand) {
- while (isa<GlobalAlias>(Ptr) && !cast<GlobalAlias>(Ptr)->isInterposable() &&
- !cast<GlobalAlias>(Ptr)->getBaseObject()->isInterposable()) {
- Ptr = cast<GlobalAlias>(Ptr)->getAliasee();
- }
- }
-
auto *NewPtrTy = cast<PointerType>(Ptr->getType());
// Preserve the address space number of the pointer.
@@ -893,8 +817,7 @@ Constant *StripPtrCastKeepAS(Constant *Ptr, bool ForLoadOperand) {
Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
ArrayRef<Constant *> Ops,
const DataLayout &DL,
- const TargetLibraryInfo *TLI,
- bool ForLoadOperand) {
+ const TargetLibraryInfo *TLI) {
const GEPOperator *InnermostGEP = GEP;
bool InBounds = GEP->isInBounds();
@@ -939,7 +862,7 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
DL.getIndexedOffsetInType(
SrcElemTy,
makeArrayRef((Value * const *)Ops.data() + 1, Ops.size() - 1)));
- Ptr = StripPtrCastKeepAS(Ptr, ForLoadOperand);
+ Ptr = StripPtrCastKeepAS(Ptr);
// If this is a GEP of a GEP, fold it all into a single GEP.
while (auto *GEP = dyn_cast<GEPOperator>(Ptr)) {
@@ -961,7 +884,7 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
Ptr = cast<Constant>(GEP->getOperand(0));
SrcElemTy = GEP->getSourceElementType();
Offset += APInt(BitWidth, DL.getIndexedOffsetInType(SrcElemTy, NestedOps));
- Ptr = StripPtrCastKeepAS(Ptr, ForLoadOperand);
+ Ptr = StripPtrCastKeepAS(Ptr);
}
// If the base value for this address is a literal integer value, fold the
@@ -1069,8 +992,7 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
return ConstantFoldCastOperand(Opcode, Ops[0], DestTy, DL);
if (auto *GEP = dyn_cast<GEPOperator>(InstOrCE)) {
- if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI,
- /*ForLoadOperand*/ false))
+ if (Constant *C = SymbolicallyEvaluateGEP(GEP, Ops, DL, TLI))
return C;
return ConstantExpr::getGetElementPtr(GEP->getSourceElementType(), Ops[0],
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll b/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll
index c5ada729b31ae..b7a1f6c78456f 100644
--- a/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/loads.ll
@@ -30,11 +30,12 @@ define i16 @test2() {
ret i16 %r
}
-; FIXME: Should be able to load through a constant addrspacecast.
define i16 @test2_addrspacecast() {
-; CHECK-LABEL: @test2_addrspacecast(
-; CHECK-NEXT: [[R:%.*]] = load i16, i16 addrspace(1)* addrspacecast (i16* bitcast ({ { i32, i8 }, i32 }* @g1 to i16*) to i16 addrspace(1)*), align 8
-; CHECK-NEXT: ret i16 [[R]]
+; LE-LABEL: @test2_addrspacecast(
+; LE-NEXT: ret i16 -16657
+;
+; BE-LABEL: @test2_addrspacecast(
+; BE-NEXT: ret i16 -8531
;
%r = load i16, i16 addrspace(1)* addrspacecast(i32* getelementptr ({{i32,i8},i32}, {{i32,i8},i32}* @g1, i32 0, i32 0, i32 0) to i16 addrspace(1)*)
ret i16 %r
@@ -246,7 +247,7 @@ define i64 @test_leading_zero_size_elems_big() {
define i64 @test_array_of_zero_size_array() {
; CHECK-LABEL: @test_array_of_zero_size_array(
-; CHECK-NEXT: ret i64 0
+; CHECK-NEXT: ret i64 undef
;
%v = load i64, i64* bitcast ([4294967295 x [0 x i32]]* @g9 to i64*)
ret i64 %v
@@ -266,8 +267,7 @@ define i32* @test_undef_aggregate() {
define {}* @test_trailing_zero_gep_index() {
; CHECK-LABEL: @test_trailing_zero_gep_index(
-; CHECK-NEXT: [[V:%.*]] = load {}*, {}** bitcast (i8* getelementptr inbounds (<{ [8 x i8], [8 x i8] }>, <{ [8 x i8], [8 x i8] }>* @g11, i64 0, i32 1, i64 0) to {}**), align 4
-; CHECK-NEXT: ret {}* [[V]]
+; CHECK-NEXT: ret {}* null
;
%v = load {}*, {}** bitcast (i8* getelementptr inbounds (<{ [8 x i8], [8 x i8] }>, <{ [8 x i8], [8 x i8] }>* @g11, i32 0, i32 1, i32 0) to {}**), align 4
ret {}* %v
diff --git a/llvm/test/Transforms/InstSimplify/load.ll b/llvm/test/Transforms/InstSimplify/load.ll
index 78c4f8bffd39c..c11eb8a0319ad 100644
--- a/llvm/test/Transforms/InstSimplify/load.ll
+++ b/llvm/test/Transforms/InstSimplify/load.ll
@@ -6,7 +6,7 @@
define i32 @crash_on_zeroinit() {
; CHECK-LABEL: @crash_on_zeroinit(
-; CHECK-NEXT: ret i32 0
+; CHECK-NEXT: ret i32 undef
;
%load = load i32, i32* bitcast ({}* @zeroinit to i32*)
ret i32 %load
More information about the llvm-commits
mailing list