[llvm] 02f74ea - [IVDescriptors] Make pointer inductions compatible with opaque pointers
Nikita Popov via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 1 12:02:12 PDT 2021
Author: Nikita Popov
Date: 2021-09-01T21:02:05+02:00
New Revision: 02f74eadbe2f56db4807dbc785a99e941a98300a
URL: https://github.com/llvm/llvm-project/commit/02f74eadbe2f56db4807dbc785a99e941a98300a
DIFF: https://github.com/llvm/llvm-project/commit/02f74eadbe2f56db4807dbc785a99e941a98300a.diff
LOG: [IVDescriptors] Make pointer inductions compatible with opaque pointers
Store the used element type in the InductionDescriptor. For typed
pointers, it remains the pointer element type. For opaque pointers,
we always use an i8 element type, such that the step is a simple
offset.
A previous version of this patch instead tried to guess the element
type from an induction GEP, but this is not reliable, as the GEP
may be hidden (see @both in iv_outside_user.ll).
Differential Revision: https://reviews.llvm.org/D104795
Added:
llvm/test/Transforms/LoopVectorize/opaque-ptr.ll
Modified:
llvm/include/llvm/Analysis/IVDescriptors.h
llvm/lib/Analysis/IVDescriptors.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 82e1b14960bd..98148f61baa1 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -329,6 +329,11 @@ class InductionDescriptor {
: Instruction::BinaryOpsEnd;
}
+ Type *getElementType() const {
+ assert(IK == IK_PtrInduction && "Only pointer induction has element type");
+ return ElementType;
+ }
+
/// Returns a reference to the type cast instructions in the induction
/// update chain, that are redundant when guarded with a runtime
/// SCEV overflow check.
@@ -340,6 +345,7 @@ class InductionDescriptor {
/// Private constructor - used by \c isInductionPHI.
InductionDescriptor(Value *Start, InductionKind K, const SCEV *Step,
BinaryOperator *InductionBinOp = nullptr,
+ Type *ElementType = nullptr,
SmallVectorImpl<Instruction *> *Casts = nullptr);
/// Start value.
@@ -350,6 +356,9 @@ class InductionDescriptor {
const SCEV *Step = nullptr;
// Instruction that advances induction variable.
BinaryOperator *InductionBinOp = nullptr;
+ // Element type for pointer induction variables.
+ // TODO: This can be dropped once support for typed pointers is removed.
+ Type *ElementType = nullptr;
// Instructions used for type-casts of the induction variable,
// that are redundant when guarded with a runtime SCEV overflow check.
SmallVector<Instruction *, 2> RedundantCasts;
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index d8b8f961136e..53375efe9764 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -963,8 +963,10 @@ RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const {
InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K,
const SCEV *Step, BinaryOperator *BOp,
+ Type *ElementType,
SmallVectorImpl<Instruction *> *Casts)
- : StartValue(Start), IK(K), Step(Step), InductionBinOp(BOp) {
+ : StartValue(Start), IK(K), Step(Step), InductionBinOp(BOp),
+ ElementType(ElementType) {
assert(IK != IK_NoInduction && "Not an induction");
// Start value type should match the induction kind and the value
@@ -992,6 +994,11 @@ InductionDescriptor::InductionDescriptor(Value *Start, InductionKind K,
InductionBinOp->getOpcode() == Instruction::FSub))) &&
"Binary opcode should be specified for FP induction");
+ if (IK == IK_PtrInduction)
+ assert(ElementType && "Pointer induction must have element type");
+ else
+ assert(!ElementType && "Non-pointer induction cannot have element type");
+
if (Casts) {
for (auto &Inst : *Casts) {
RedundantCasts.push_back(Inst);
@@ -1239,8 +1246,6 @@ bool InductionDescriptor::isInductionPHI(
BasicBlock *Latch = AR->getLoop()->getLoopLatch();
if (!Latch)
return false;
- BinaryOperator *BOp =
- dyn_cast<BinaryOperator>(Phi->getIncomingValueForBlock(Latch));
const SCEV *Step = AR->getStepRecurrence(*SE);
// Calculate the pointer stride and check if it is consecutive.
@@ -1250,8 +1255,10 @@ bool InductionDescriptor::isInductionPHI(
return false;
if (PhiTy->isIntegerTy()) {
+ BinaryOperator *BOp =
+ dyn_cast<BinaryOperator>(Phi->getIncomingValueForBlock(Latch));
D = InductionDescriptor(StartValue, IK_IntInduction, Step, BOp,
- CastsToIgnore);
+ /* ElementType */ nullptr, CastsToIgnore);
return true;
}
@@ -1260,15 +1267,16 @@ bool InductionDescriptor::isInductionPHI(
if (!ConstStep)
return false;
- ConstantInt *CV = ConstStep->getValue();
- Type *PointerElementType = PhiTy->getPointerElementType();
- // The pointer stride cannot be determined if the pointer element type is not
- // sized.
- if (!PointerElementType->isSized())
+ // Always use i8 element type for opaque pointer inductions.
+ PointerType *PtrTy = cast<PointerType>(PhiTy);
+ Type *ElementType = PtrTy->isOpaque() ? Type::getInt8Ty(PtrTy->getContext())
+ : PtrTy->getElementType();
+ if (!ElementType->isSized())
return false;
+ ConstantInt *CV = ConstStep->getValue();
const DataLayout &DL = Phi->getModule()->getDataLayout();
- int64_t Size = static_cast<int64_t>(DL.getTypeAllocSize(PointerElementType));
+ int64_t Size = static_cast<int64_t>(DL.getTypeAllocSize(ElementType));
if (!Size)
return false;
@@ -1277,6 +1285,7 @@ bool InductionDescriptor::isInductionPHI(
return false;
auto *StepValue =
SE->getConstant(CV->getType(), CVSize / Size, true /* signed */);
- D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue, BOp);
+ D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue,
+ /* BinOp */ nullptr, ElementType);
return true;
}
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 260e1824870c..06f05da2a27c 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3448,7 +3448,7 @@ Value *InnerLoopVectorizer::emitTransformedIndex(
assert(isa<SCEVConstant>(Step) &&
"Expected constant step for pointer induction");
return B.CreateGEP(
- StartValue->getType()->getPointerElementType(), StartValue,
+ ID.getElementType(), StartValue,
CreateMul(Index,
Exp.expandCodeFor(Step, Index->getType()->getScalarType(),
GetInsertPoint())));
@@ -4811,7 +4811,7 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
Value *NumUnrolledElems =
Builder.CreateMul(RuntimeVF, ConstantInt::get(PhiType, State.UF));
Value *InductionGEP = GetElementPtrInst::Create(
- ScStValueType->getPointerElementType(), NewPointerPhi,
+ II.getElementType(), NewPointerPhi,
Builder.CreateMul(ScalarStepValue, NumUnrolledElems), "ptr.ind",
InductionLoc);
NewPointerPhi->addIncoming(InductionGEP, LoopLatch);
@@ -4830,7 +4830,7 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
Builder.CreateAdd(StartOffset, Builder.CreateStepVector(VecPhiType));
Value *GEP = Builder.CreateGEP(
- ScStValueType->getPointerElementType(), NewPointerPhi,
+ II.getElementType(), NewPointerPhi,
Builder.CreateMul(
StartOffset, Builder.CreateVectorSplat(State.VF, ScalarStepValue),
"vector.gep"));
diff --git a/llvm/test/Transforms/LoopVectorize/opaque-ptr.ll b/llvm/test/Transforms/LoopVectorize/opaque-ptr.ll
new file mode 100644
index 000000000000..f47f60df910e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/opaque-ptr.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-vectorize -force-vector-width=2 < %s | FileCheck %s
+
+; TODO: This still crashes with inbounds on the GEPs.
+define void @test(ptr %p1.start, ptr %p2.start, ptr %p1.end) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[LOOP:%.*]]
+; CHECK: loop:
+; CHECK-NEXT: [[P1:%.*]] = phi ptr [ [[P1_START:%.*]], [[ENTRY:%.*]] ], [ [[P1_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[P2:%.*]] = phi ptr [ [[P2_START:%.*]], [[ENTRY]] ], [ [[P2_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT: [[P1_VAL:%.*]] = load float, ptr [[P1]], align 4
+; CHECK-NEXT: [[P2_VAL:%.*]] = load float, ptr [[P2]], align 4
+; CHECK-NEXT: [[SUM:%.*]] = fadd float [[P1_VAL]], [[P2_VAL]]
+; CHECK-NEXT: store float [[SUM]], ptr [[P1]], align 4
+; CHECK-NEXT: [[P1_NEXT]] = getelementptr float, ptr [[P1]], i64 1
+; CHECK-NEXT: [[P2_NEXT]] = getelementptr float, ptr [[P2]], i64 1
+; CHECK-NEXT: [[C:%.*]] = icmp ne ptr [[P1_NEXT]], [[P1_END:%.*]]
+; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %p1 = phi ptr [ %p1.start, %entry ], [ %p1.next, %loop ]
+ %p2 = phi ptr [ %p2.start, %entry ], [ %p2.next, %loop ]
+ %p1.val = load float, ptr %p1
+ %p2.val = load float, ptr %p2
+ %sum = fadd float %p1.val, %p2.val
+ store float %sum, ptr %p1
+ %p1.next = getelementptr float, ptr %p1, i64 1
+ %p2.next = getelementptr float, ptr %p2, i64 1
+ %c = icmp ne ptr %p1.next, %p1.end
+ br i1 %c, label %loop, label %exit
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list