[llvm] [SCEV] Don't use non-deterministic constant folding for trip counts (PR #90942)
Nikita Popov via llvm-commits
llvm-commits at lists.llvm.org
Wed May 15 19:23:32 PDT 2024
https://github.com/nikic updated https://github.com/llvm/llvm-project/pull/90942
>From 5eb9f19ce09c16b3d021194caf369852719b6519 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov at redhat.com>
Date: Fri, 3 May 2024 15:10:54 +0900
Subject: [PATCH 1/4] [SCEV] Add tests for #89885 (NFC)
---
.../ScalarEvolution/exhaustive-trip-counts.ll | 54 +++++++++++++++++++
1 file changed, 54 insertions(+)
diff --git a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll
index 21237f4266933..200009d1277f5 100644
--- a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll
+++ b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll
@@ -27,4 +27,58 @@ for.cond.cleanup:
ret void
}
+
+define i64 @test_fp_libcall() {
+; CHECK-LABEL: 'test_fp_libcall'
+; CHECK-NEXT: Determining loop execution counts for: @test_fp_libcall
+; CHECK-NEXT: Loop %loop: backedge-taken count is i32 90
+; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 90
+; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 90
+; CHECK-NEXT: Loop %loop: Trip multiple is 91
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %fv = phi double [ 1.000000e+00, %entry ], [ %fv.next, %loop ]
+ call void @use(double %fv)
+ %fv.next = call double @llvm.sin.f64(double %fv)
+ %iv.next = add i64 %iv, 1
+ %fcmp = fcmp une double %fv, 0x3FC6BA15EE8460B0
+ br i1 %fcmp, label %loop, label %exit
+
+exit:
+ ret i64 %iv
+}
+
+define i64 @test_nan_sign() {
+; CHECK-LABEL: 'test_nan_sign'
+; CHECK-NEXT: Determining loop execution counts for: @test_nan_sign
+; CHECK-NEXT: Loop %loop: backedge-taken count is i32 46
+; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 46
+; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 46
+; CHECK-NEXT: Loop %loop: Trip multiple is 47
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %fv = phi double [ -1.000000e+00, %entry ], [ %fv.next, %loop ]
+ call void @use(double %fv)
+ %a = fsub double %fv, 0x7F86C16C16C16C16
+ %b = fadd double %a, %a
+ %fv.next = fsub double %b, %a
+ %iv.next = add i64 %iv, 1
+ %fv.bc = bitcast double %fv to i64
+ %icmp = icmp slt i64 %fv.bc, 0
+ br i1 %icmp, label %loop, label %exit
+
+exit:
+ ret i64 %iv
+}
+
declare void @dummy()
+declare void @use(double %i)
+declare double @llvm.sin.f64(double)
>From d75561031fa6fb151576da8f4be55222a6a5e608 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov at redhat.com>
Date: Fri, 3 May 2024 14:40:21 +0900
Subject: [PATCH 2/4] [SCEV] Do not use non-deterministic constant folding
results for exhaustive trip counts
---
llvm/include/llvm/Analysis/ConstantFolding.h | 14 +++++--
llvm/lib/Analysis/ConstantFolding.cpp | 42 ++++++++++++++-----
llvm/lib/Analysis/ScalarEvolution.cpp | 6 ++-
.../ScalarEvolution/exhaustive-trip-counts.ll | 19 +++++----
4 files changed, 56 insertions(+), 25 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index c54b1e8f01d2b..3269c7198cb47 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -68,9 +68,15 @@ Constant *ConstantFoldConstant(const Constant *C, const DataLayout &DL,
/// fold instructions like loads and stores, which have no constant expression
/// form.
///
+/// In some cases, constant folding may return one value chosen from a set of
+/// multiple legal return values. Using such a result is usually only valid if
+/// all uses of the original operation are replaced by the constant-folded
+/// result. The \p AllowNonDeterministic parameter controls whether this is
+/// allowed.
Constant *ConstantFoldInstOperands(Instruction *I, ArrayRef<Constant *> Ops,
const DataLayout &DL,
- const TargetLibraryInfo *TLI = nullptr);
+ const TargetLibraryInfo *TLI = nullptr,
+ bool AllowNonDeterministic = true);
/// Attempt to constant fold a compare instruction (icmp/fcmp) with the
/// specified operands. Returns null or a constant expression of the specified
@@ -95,7 +101,8 @@ Constant *ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS,
/// Returns null or a constant expression of the specified operands on failure.
Constant *ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS,
Constant *RHS, const DataLayout &DL,
- const Instruction *I);
+ const Instruction *I,
+ bool AllowNonDeterministic = true);
/// Attempt to flush float point constant according to denormal mode set in the
/// instruction's parent function attributes. If so, return a zero with the
@@ -190,7 +197,8 @@ bool canConstantFoldCallTo(const CallBase *Call, const Function *F);
/// with the specified arguments, returning null if unsuccessful.
Constant *ConstantFoldCall(const CallBase *Call, Function *F,
ArrayRef<Constant *> Operands,
- const TargetLibraryInfo *TLI = nullptr);
+ const TargetLibraryInfo *TLI = nullptr,
+ bool AllowNonDeterministic = true);
Constant *ConstantFoldBinaryIntrinsic(Intrinsic::ID ID, Constant *LHS,
Constant *RHS, Type *Ty,
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 046a769453808..732b2b7a30ce6 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -992,7 +992,8 @@ Constant *SymbolicallyEvaluateGEP(const GEPOperator *GEP,
Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
ArrayRef<Constant *> Ops,
const DataLayout &DL,
- const TargetLibraryInfo *TLI) {
+ const TargetLibraryInfo *TLI,
+ bool AllowNonDeterministic) {
Type *DestTy = InstOrCE->getType();
if (Instruction::isUnaryOp(Opcode))
@@ -1011,7 +1012,8 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
// TODO: If a constant expression is being folded rather than an
// instruction, denormals will not be flushed/treated as zero
if (const auto *I = dyn_cast<Instruction>(InstOrCE)) {
- return ConstantFoldFPInstOperands(Opcode, Ops[0], Ops[1], DL, I);
+ return ConstantFoldFPInstOperands(Opcode, Ops[0], Ops[1], DL, I,
+ AllowNonDeterministic);
}
}
return ConstantFoldBinaryOpOperands(Opcode, Ops[0], Ops[1], DL);
@@ -1053,7 +1055,8 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
if (auto *F = dyn_cast<Function>(Ops.back())) {
const auto *Call = cast<CallBase>(InstOrCE);
if (canConstantFoldCallTo(Call, F))
- return ConstantFoldCall(Call, F, Ops.slice(0, Ops.size() - 1), TLI);
+ return ConstantFoldCall(Call, F, Ops.slice(0, Ops.size() - 1), TLI,
+ AllowNonDeterministic);
}
return nullptr;
case Instruction::Select:
@@ -1114,8 +1117,8 @@ ConstantFoldConstantImpl(const Constant *C, const DataLayout &DL,
}
if (auto *CE = dyn_cast<ConstantExpr>(C)) {
- if (Constant *Res =
- ConstantFoldInstOperandsImpl(CE, CE->getOpcode(), Ops, DL, TLI))
+ if (Constant *Res = ConstantFoldInstOperandsImpl(
+ CE, CE->getOpcode(), Ops, DL, TLI, /*AllowNonDeterministic=*/true))
return Res;
return const_cast<Constant *>(C);
}
@@ -1183,8 +1186,10 @@ Constant *llvm::ConstantFoldConstant(const Constant *C, const DataLayout &DL,
Constant *llvm::ConstantFoldInstOperands(Instruction *I,
ArrayRef<Constant *> Ops,
const DataLayout &DL,
- const TargetLibraryInfo *TLI) {
- return ConstantFoldInstOperandsImpl(I, I->getOpcode(), Ops, DL, TLI);
+ const TargetLibraryInfo *TLI,
+ bool AllowNonDeterministic) {
+ return ConstantFoldInstOperandsImpl(I, I->getOpcode(), Ops, DL, TLI,
+ AllowNonDeterministic);
}
Constant *llvm::ConstantFoldCompareInstOperands(
@@ -1357,7 +1362,8 @@ Constant *llvm::FlushFPConstant(Constant *Operand, const Instruction *I,
Constant *llvm::ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS,
Constant *RHS, const DataLayout &DL,
- const Instruction *I) {
+ const Instruction *I,
+ bool AllowNonDeterministic) {
if (Instruction::isBinaryOp(Opcode)) {
// Flush denormal inputs if needed.
Constant *Op0 = FlushFPConstant(LHS, I, /* IsOutput */ false);
@@ -1373,7 +1379,15 @@ Constant *llvm::ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS,
return nullptr;
// Flush denormal output if needed.
- return FlushFPConstant(C, I, /* IsOutput */ true);
+ C = FlushFPConstant(C, I, /* IsOutput */ true);
+ if (!C)
+ return nullptr;
+
+ // The precise NaN value is non-deterministic.
+ if (!AllowNonDeterministic && C->isNaN())
+ return nullptr;
+
+ return C;
}
// If instruction lacks a parent/function and the denormal mode cannot be
// determined, use the default (IEEE).
@@ -3401,7 +3415,8 @@ Constant *llvm::ConstantFoldBinaryIntrinsic(Intrinsic::ID ID, Constant *LHS,
Constant *llvm::ConstantFoldCall(const CallBase *Call, Function *F,
ArrayRef<Constant *> Operands,
- const TargetLibraryInfo *TLI) {
+ const TargetLibraryInfo *TLI,
+ bool AllowNonDeterministic) {
if (Call->isNoBuiltin())
return nullptr;
if (!F->hasName())
@@ -3417,8 +3432,13 @@ Constant *llvm::ConstantFoldCall(const CallBase *Call, Function *F,
return nullptr;
}
- StringRef Name = F->getName();
+ // Conservatively assume that floating-point libcalls may be
+ // non-deterministic.
Type *Ty = F->getReturnType();
+ if (!AllowNonDeterministic && Ty->isFPOrFPVectorTy())
+ return nullptr;
+
+ StringRef Name = F->getName();
if (auto *FVTy = dyn_cast<FixedVectorType>(Ty))
return ConstantFoldFixedVectorCall(
Name, IID, FVTy, Operands, F->getParent()->getDataLayout(), TLI, Call);
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 254d79183a1e9..704f92669a117 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -9540,7 +9540,8 @@ static Constant *EvaluateExpression(Value *V, const Loop *L,
Operands[i] = C;
}
- return ConstantFoldInstOperands(I, Operands, DL, TLI);
+ return ConstantFoldInstOperands(I, Operands, DL, TLI,
+ /*AllowNonDeterministic=*/false);
}
@@ -10031,7 +10032,8 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
Constant *C = nullptr;
const DataLayout &DL = getDataLayout();
- C = ConstantFoldInstOperands(I, Operands, DL, &TLI);
+ C = ConstantFoldInstOperands(I, Operands, DL, &TLI,
+ /*AllowNonDeterministic=*/false);
if (!C)
return V;
return getSCEV(C);
diff --git a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll
index 200009d1277f5..3975e7173e7b7 100644
--- a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll
+++ b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll
@@ -27,14 +27,14 @@ for.cond.cleanup:
ret void
}
-
+; Do not compute exhaustive trip count based on FP libcalls, as their exact
+; return value may not be specified.
define i64 @test_fp_libcall() {
; CHECK-LABEL: 'test_fp_libcall'
; CHECK-NEXT: Determining loop execution counts for: @test_fp_libcall
-; CHECK-NEXT: Loop %loop: backedge-taken count is i32 90
-; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 90
-; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 90
-; CHECK-NEXT: Loop %loop: Trip multiple is 91
+; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count.
;
entry:
br label %loop
@@ -52,13 +52,14 @@ exit:
ret i64 %iv
}
+; Do not compute exhaustive trip count based on FP constant folding resulting
+; in NaN values, as we don't specify which NaN exactly is returned.
define i64 @test_nan_sign() {
; CHECK-LABEL: 'test_nan_sign'
; CHECK-NEXT: Determining loop execution counts for: @test_nan_sign
-; CHECK-NEXT: Loop %loop: backedge-taken count is i32 46
-; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i32 46
-; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i32 46
-; CHECK-NEXT: Loop %loop: Trip multiple is 47
+; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count.
;
entry:
br label %loop
>From 941fbc7ee8ba8863788e2acc298bfbb07d4a2e57 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov at redhat.com>
Date: Thu, 16 May 2024 10:55:14 +0900
Subject: [PATCH 3/4] Mention NaN as an example
---
llvm/include/llvm/Analysis/ConstantFolding.h | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index 3269c7198cb47..58b38fb8b0367 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -69,7 +69,8 @@ Constant *ConstantFoldConstant(const Constant *C, const DataLayout &DL,
/// form.
///
/// In some cases, constant folding may return one value chosen from a set of
-/// multiple legal return values. Using such a result is usually only valid if
+/// multiple legal return values. For example, the exact bit pattern of NaN
+/// results is not guaranteed. Using such a result is usually only valid if
/// all uses of the original operation are replaced by the constant-folded
/// result. The \p AllowNonDeterministic parameter controls whether this is
/// allowed.
>From ab175147b531eb09f20822c0e637eedfcd7f47f7 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov at redhat.com>
Date: Thu, 16 May 2024 11:23:00 +0900
Subject: [PATCH 4/4] Also disable folding of operations with certain FMF flags
---
llvm/lib/Analysis/ConstantFolding.cpp | 9 ++
.../ScalarEvolution/exhaustive-trip-counts.ll | 97 +++++++++++++++++++
2 files changed, 106 insertions(+)
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 732b2b7a30ce6..524e84f3f3ded 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1373,6 +1373,15 @@ Constant *llvm::ConstantFoldFPInstOperands(unsigned Opcode, Constant *LHS,
if (!Op1)
return nullptr;
+ // If nsz or an algebraic FMF flag is set, the result of the FP operation
+ // may change due to future optimization. Don't constant fold them if
+ // non-deterministic results are not allowed.
+ if (!AllowNonDeterministic)
+ if (auto *FP = dyn_cast_or_null<FPMathOperator>(I))
+ if (FP->hasNoSignedZeros() || FP->hasAllowReassoc() ||
+ FP->hasAllowContract() || FP->hasAllowReciprocal())
+ return nullptr;
+
// Calculate constant result.
Constant *C = ConstantFoldBinaryOpOperands(Opcode, Op0, Op1, DL);
if (!C)
diff --git a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll
index 3975e7173e7b7..cc08fa5fc7d87 100644
--- a/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll
+++ b/llvm/test/Analysis/ScalarEvolution/exhaustive-trip-counts.ll
@@ -80,6 +80,103 @@ exit:
ret i64 %iv
}
+; Do not compute exhaustive trip count based on FP constant folding if the
+; involved operation has nsz or one of the algebraic FMF flags (reassoc, arcp,
+; contract) set. The examples in the following are dummies and don't illustrate
+; real cases where FMF transforms could cause issues.
+
+define i64 @test_fp_nsz() {
+; CHECK-LABEL: 'test_fp_nsz'
+; CHECK-NEXT: Determining loop execution counts for: @test_fp_nsz
+; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %fv = phi double [ 1.000000e+00, %entry ], [ %fv.next, %loop ]
+ call void @use(double %fv)
+ %fv.next = fadd nsz double %fv, 1.0
+ %iv.next = add i64 %iv, 1
+ %fcmp = fcmp une double %fv, 100.0
+ br i1 %fcmp, label %loop, label %exit
+
+exit:
+ ret i64 %iv
+}
+
+define i64 @test_fp_reassoc() {
+; CHECK-LABEL: 'test_fp_reassoc'
+; CHECK-NEXT: Determining loop execution counts for: @test_fp_reassoc
+; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %fv = phi double [ 1.000000e+00, %entry ], [ %fv.next, %loop ]
+ call void @use(double %fv)
+ %fv.next = fadd reassoc double %fv, 1.0
+ %iv.next = add i64 %iv, 1
+ %fcmp = fcmp une double %fv, 100.0
+ br i1 %fcmp, label %loop, label %exit
+
+exit:
+ ret i64 %iv
+}
+
+define i64 @test_fp_arcp() {
+; CHECK-LABEL: 'test_fp_arcp'
+; CHECK-NEXT: Determining loop execution counts for: @test_fp_arcp
+; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %fv = phi double [ 1.000000e+00, %entry ], [ %fv.next, %loop ]
+ call void @use(double %fv)
+ %fv.next = fadd arcp double %fv, 1.0
+ %iv.next = add i64 %iv, 1
+ %fcmp = fcmp une double %fv, 100.0
+ br i1 %fcmp, label %loop, label %exit
+
+exit:
+ ret i64 %iv
+}
+
+define i64 @test_fp_contract() {
+; CHECK-LABEL: 'test_fp_contract'
+; CHECK-NEXT: Determining loop execution counts for: @test_fp_contract
+; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count.
+; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count.
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %fv = phi double [ 1.000000e+00, %entry ], [ %fv.next, %loop ]
+ call void @use(double %fv)
+ %fv.next = fadd contract double %fv, 1.0
+ %iv.next = add i64 %iv, 1
+ %fcmp = fcmp une double %fv, 100.0
+ br i1 %fcmp, label %loop, label %exit
+
+exit:
+ ret i64 %iv
+}
+
declare void @dummy()
declare void @use(double %i)
declare double @llvm.sin.f64(double)
More information about the llvm-commits
mailing list