[llvm] [TTI][BPF]: Undo specific transform-preventing passes and add one TTI hook (PR #97073)
Dmitry Spikhalsky via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 28 08:33:33 PDT 2024
https://github.com/Spikhalskiy created https://github.com/llvm/llvm-project/pull/97073
Adaptation of the original https://reviews.llvm.org/D147968
>From 6a2dff553f9c86c64906144af0a5b1d18fa3befb Mon Sep 17 00:00:00 2001
From: Dmitry Spikhalsky <dmitry at spikhalskiy.com>
Date: Fri, 28 Jun 2024 10:25:27 -0400
Subject: [PATCH] [TTI][BPF]: Undo specific transform-preventing passes and add
one TTI hook
Adaptation of the original https://reviews.llvm.org/D147968
---
.../llvm/Analysis/TargetTransformInfo.h | 9 +
.../llvm/Analysis/TargetTransformInfoImpl.h | 2 +
llvm/include/llvm/IR/IntrinsicsBPF.td | 3 -
.../Transforms/InstCombine/InstCombiner.h | 1 +
.../include/llvm/Transforms/Utils/LoopUtils.h | 8 +-
llvm/lib/Analysis/TargetTransformInfo.cpp | 4 +
llvm/lib/Target/BPF/BPF.h | 5 -
llvm/lib/Target/BPF/BPFAdjustOpt.cpp | 376 ------------------
llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp | 48 +--
llvm/lib/Target/BPF/BPFTargetMachine.cpp | 4 -
llvm/lib/Target/BPF/BPFTargetTransformInfo.h | 3 +
llvm/lib/Target/BPF/CMakeLists.txt | 1 -
.../InstCombine/InstCombineAndOrXor.cpp | 4 +
llvm/lib/Transforms/Scalar/LICM.cpp | 27 +-
llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 3 +
llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll | 14 +-
llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll | 10 +-
llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll | 12 +-
llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll | 12 +-
.../CodeGen/BPF/adjust-opt-speculative1.ll | 17 +-
.../CodeGen/BPF/adjust-opt-speculative2.ll | 22 +-
21 files changed, 69 insertions(+), 516 deletions(-)
delete mode 100644 llvm/lib/Target/BPF/BPFAdjustOpt.cpp
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index f55f21c94a85a..7e2f75d8c4cde 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1771,6 +1771,10 @@ class TargetTransformInfo {
/// \return The maximum number of function arguments the target supports.
unsigned getMaxNumArgs() const;
+ /// \returns True if the target wants to preserve range information to make
+ /// later code verification easier.
+ bool needsPreserveRangeInfoInVerification() const;
+
/// @}
private:
@@ -2160,6 +2164,7 @@ class TargetTransformInfo::Concept {
getVPLegalizationStrategy(const VPIntrinsic &PI) const = 0;
virtual bool hasArmWideBranch(bool Thumb) const = 0;
virtual unsigned getMaxNumArgs() const = 0;
+ virtual bool needsPreserveRangeInfoInVerification() const = 0;
};
template <typename T>
@@ -2922,6 +2927,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
unsigned getMaxNumArgs() const override {
return Impl.getMaxNumArgs();
}
+
+ bool needsPreserveRangeInfoInVerification() const override {
+ return Impl.needsPreserveRangeInfoInVerification();
+ }
};
template <typename T>
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 7828bdc1f1f43..456332120368d 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -953,6 +953,8 @@ class TargetTransformInfoImplBase {
unsigned getMaxNumArgs() const { return UINT_MAX; }
+ bool needsPreserveRangeInfoInVerification() const { return false; }
+
protected:
// Obtain the minimum required size to hold the value (without the sign)
// In case of a vector it returns the min required size for one element.
diff --git a/llvm/include/llvm/IR/IntrinsicsBPF.td b/llvm/include/llvm/IR/IntrinsicsBPF.td
index c7ec0916f1d1f..781305851cea1 100644
--- a/llvm/include/llvm/IR/IntrinsicsBPF.td
+++ b/llvm/include/llvm/IR/IntrinsicsBPF.td
@@ -34,9 +34,6 @@ let TargetPrefix = "bpf" in { // All intrinsics start with "llvm.bpf."
[IntrNoMem]>;
def int_bpf_passthrough : ClangBuiltin<"__builtin_bpf_passthrough">,
Intrinsic<[llvm_any_ty], [llvm_i32_ty, llvm_any_ty], [IntrNoMem]>;
- def int_bpf_compare : ClangBuiltin<"__builtin_bpf_compare">,
- Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_anyint_ty, llvm_anyint_ty],
- [IntrNoMem]>;
def int_bpf_getelementptr_and_load : ClangBuiltin<"__builtin_bpf_getelementptr_and_load">,
Intrinsic<[llvm_any_ty],
[llvm_ptr_ty, // base ptr for getelementptr
diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
index 855d1aeddfaee..29db6834f3b89 100644
--- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
+++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h
@@ -45,6 +45,7 @@ class TargetTransformInfo;
/// This class provides both the logic to recursively visit instructions and
/// combine them.
class LLVM_LIBRARY_VISIBILITY InstCombiner {
+protected:
/// Only used to call target specific intrinsic combining.
/// It must **NOT** be used for any other purpose, as InstCombine is a
/// target-independent canonicalization transform.
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
index 345e09dce0b2b..9c61c77cd7a64 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h
@@ -173,10 +173,10 @@ bool sinkRegionForLoopNest(DomTreeNode *, AAResults *, LoopInfo *,
/// \p AllowSpeculation is whether values should be hoisted even if they are not
/// guaranteed to execute in the loop, but are safe to speculatively execute.
bool hoistRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *,
- AssumptionCache *, TargetLibraryInfo *, Loop *,
- MemorySSAUpdater &, ScalarEvolution *, ICFLoopSafetyInfo *,
- SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *, bool,
- bool AllowSpeculation);
+ AssumptionCache *, TargetLibraryInfo *, TargetTransformInfo *,
+ Loop *, MemorySSAUpdater &, ScalarEvolution *,
+ ICFLoopSafetyInfo *, SinkAndHoistLICMFlags &,
+ OptimizationRemarkEmitter *, bool, bool AllowSpeculation);
/// Return true if the induction variable \p IV in a Loop whose latch is
/// \p LatchBlock would become dead if the exit test \p Cond were removed.
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index e6a4a2e1f6110..0e8d6621af6c8 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1309,6 +1309,10 @@ unsigned TargetTransformInfo::getMaxNumArgs() const {
return TTIImpl->getMaxNumArgs();
}
+bool TargetTransformInfo::needsPreserveRangeInfoInVerification() const {
+ return TTIImpl->needsPreserveRangeInfoInVerification();
+}
+
bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
return TTIImpl->shouldExpandReduction(II);
}
diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h
index 694d7bacf6421..84c753e2dc4fc 100644
--- a/llvm/lib/Target/BPF/BPF.h
+++ b/llvm/lib/Target/BPF/BPF.h
@@ -74,11 +74,6 @@ class BPFASpaceCastSimplifyPass
static bool isRequired() { return true; }
};
-class BPFAdjustOptPass : public PassInfoMixin<BPFAdjustOptPass> {
-public:
- PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-};
-
class BPFPreserveStaticOffsetPass
: public PassInfoMixin<BPFPreserveStaticOffsetPass> {
bool AllowPartial;
diff --git a/llvm/lib/Target/BPF/BPFAdjustOpt.cpp b/llvm/lib/Target/BPF/BPFAdjustOpt.cpp
deleted file mode 100644
index 4ab0cbcc92477..0000000000000
--- a/llvm/lib/Target/BPF/BPFAdjustOpt.cpp
+++ /dev/null
@@ -1,376 +0,0 @@
-//===---------------- BPFAdjustOpt.cpp - Adjust Optimization --------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Adjust optimization to make the code more kernel verifier friendly.
-//
-//===----------------------------------------------------------------------===//
-
-#include "BPF.h"
-#include "BPFCORE.h"
-#include "BPFTargetMachine.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicsBPF.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/PatternMatch.h"
-#include "llvm/IR/Type.h"
-#include "llvm/IR/User.h"
-#include "llvm/IR/Value.h"
-#include "llvm/Pass.h"
-#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-
-#define DEBUG_TYPE "bpf-adjust-opt"
-
-using namespace llvm;
-using namespace llvm::PatternMatch;
-
-static cl::opt<bool>
- DisableBPFserializeICMP("bpf-disable-serialize-icmp", cl::Hidden,
- cl::desc("BPF: Disable Serializing ICMP insns."),
- cl::init(false));
-
-static cl::opt<bool> DisableBPFavoidSpeculation(
- "bpf-disable-avoid-speculation", cl::Hidden,
- cl::desc("BPF: Disable Avoiding Speculative Code Motion."),
- cl::init(false));
-
-namespace {
-class BPFAdjustOptImpl {
- struct PassThroughInfo {
- Instruction *Input;
- Instruction *UsedInst;
- uint32_t OpIdx;
- PassThroughInfo(Instruction *I, Instruction *U, uint32_t Idx)
- : Input(I), UsedInst(U), OpIdx(Idx) {}
- };
-
-public:
- BPFAdjustOptImpl(Module *M) : M(M) {}
-
- bool run();
-
-private:
- Module *M;
- SmallVector<PassThroughInfo, 16> PassThroughs;
-
- bool adjustICmpToBuiltin();
- void adjustBasicBlock(BasicBlock &BB);
- bool serializeICMPCrossBB(BasicBlock &BB);
- void adjustInst(Instruction &I);
- bool serializeICMPInBB(Instruction &I);
- bool avoidSpeculation(Instruction &I);
- bool insertPassThrough();
-};
-
-} // End anonymous namespace
-
-bool BPFAdjustOptImpl::run() {
- bool Changed = adjustICmpToBuiltin();
-
- for (Function &F : *M)
- for (auto &BB : F) {
- adjustBasicBlock(BB);
- for (auto &I : BB)
- adjustInst(I);
- }
- return insertPassThrough() || Changed;
-}
-
-// Commit acabad9ff6bf ("[InstCombine] try to canonicalize icmp with
-// trunc op into mask and cmp") added a transformation to
-// convert "(conv)a < power_2_const" to "a & <const>" in certain
-// cases and bpf kernel verifier has to handle the resulted code
-// conservatively and this may reject otherwise legitimate program.
-// Here, we change related icmp code to a builtin which will
-// be restored to original icmp code later to prevent that
-// InstCombine transformatin.
-bool BPFAdjustOptImpl::adjustICmpToBuiltin() {
- bool Changed = false;
- ICmpInst *ToBeDeleted = nullptr;
- for (Function &F : *M)
- for (auto &BB : F)
- for (auto &I : BB) {
- if (ToBeDeleted) {
- ToBeDeleted->eraseFromParent();
- ToBeDeleted = nullptr;
- }
-
- auto *Icmp = dyn_cast<ICmpInst>(&I);
- if (!Icmp)
- continue;
-
- Value *Op0 = Icmp->getOperand(0);
- if (!isa<TruncInst>(Op0))
- continue;
-
- auto ConstOp1 = dyn_cast<ConstantInt>(Icmp->getOperand(1));
- if (!ConstOp1)
- continue;
-
- auto ConstOp1Val = ConstOp1->getValue().getZExtValue();
- auto Op = Icmp->getPredicate();
- if (Op == ICmpInst::ICMP_ULT || Op == ICmpInst::ICMP_UGE) {
- if ((ConstOp1Val - 1) & ConstOp1Val)
- continue;
- } else if (Op == ICmpInst::ICMP_ULE || Op == ICmpInst::ICMP_UGT) {
- if (ConstOp1Val & (ConstOp1Val + 1))
- continue;
- } else {
- continue;
- }
-
- Constant *Opcode =
- ConstantInt::get(Type::getInt32Ty(BB.getContext()), Op);
- Function *Fn = Intrinsic::getDeclaration(
- M, Intrinsic::bpf_compare, {Op0->getType(), ConstOp1->getType()});
- auto *NewInst = CallInst::Create(Fn, {Opcode, Op0, ConstOp1});
- NewInst->insertBefore(&I);
- Icmp->replaceAllUsesWith(NewInst);
- Changed = true;
- ToBeDeleted = Icmp;
- }
-
- return Changed;
-}
-
-bool BPFAdjustOptImpl::insertPassThrough() {
- for (auto &Info : PassThroughs) {
- auto *CI = BPFCoreSharedInfo::insertPassThrough(
- M, Info.UsedInst->getParent(), Info.Input, Info.UsedInst);
- Info.UsedInst->setOperand(Info.OpIdx, CI);
- }
-
- return !PassThroughs.empty();
-}
-
-// To avoid combining conditionals in the same basic block by
-// instrcombine optimization.
-bool BPFAdjustOptImpl::serializeICMPInBB(Instruction &I) {
- // For:
- // comp1 = icmp <opcode> ...;
- // comp2 = icmp <opcode> ...;
- // ... or comp1 comp2 ...
- // changed to:
- // comp1 = icmp <opcode> ...;
- // comp2 = icmp <opcode> ...;
- // new_comp1 = __builtin_bpf_passthrough(seq_num, comp1)
- // ... or new_comp1 comp2 ...
- Value *Op0, *Op1;
- // Use LogicalOr (accept `or i1` as well as `select i1 Op0, true, Op1`)
- if (!match(&I, m_LogicalOr(m_Value(Op0), m_Value(Op1))))
- return false;
- auto *Icmp1 = dyn_cast<ICmpInst>(Op0);
- if (!Icmp1)
- return false;
- auto *Icmp2 = dyn_cast<ICmpInst>(Op1);
- if (!Icmp2)
- return false;
-
- Value *Icmp1Op0 = Icmp1->getOperand(0);
- Value *Icmp2Op0 = Icmp2->getOperand(0);
- if (Icmp1Op0 != Icmp2Op0)
- return false;
-
- // Now we got two icmp instructions which feed into
- // an "or" instruction.
- PassThroughInfo Info(Icmp1, &I, 0);
- PassThroughs.push_back(Info);
- return true;
-}
-
-// To avoid combining conditionals in the same basic block by
-// instrcombine optimization.
-bool BPFAdjustOptImpl::serializeICMPCrossBB(BasicBlock &BB) {
- // For:
- // B1:
- // comp1 = icmp <opcode> ...;
- // if (comp1) goto B2 else B3;
- // B2:
- // comp2 = icmp <opcode> ...;
- // if (comp2) goto B4 else B5;
- // B4:
- // ...
- // changed to:
- // B1:
- // comp1 = icmp <opcode> ...;
- // comp1 = __builtin_bpf_passthrough(seq_num, comp1);
- // if (comp1) goto B2 else B3;
- // B2:
- // comp2 = icmp <opcode> ...;
- // if (comp2) goto B4 else B5;
- // B4:
- // ...
-
- // Check basic predecessors, if two of them (say B1, B2) are using
- // icmp instructions to generate conditions and one is the predesessor
- // of another (e.g., B1 is the predecessor of B2). Add a passthrough
- // barrier after icmp inst of block B1.
- BasicBlock *B2 = BB.getSinglePredecessor();
- if (!B2)
- return false;
-
- BasicBlock *B1 = B2->getSinglePredecessor();
- if (!B1)
- return false;
-
- Instruction *TI = B2->getTerminator();
- auto *BI = dyn_cast<BranchInst>(TI);
- if (!BI || !BI->isConditional())
- return false;
- auto *Cond = dyn_cast<ICmpInst>(BI->getCondition());
- if (!Cond || B2->getFirstNonPHI() != Cond)
- return false;
- Value *B2Op0 = Cond->getOperand(0);
- auto Cond2Op = Cond->getPredicate();
-
- TI = B1->getTerminator();
- BI = dyn_cast<BranchInst>(TI);
- if (!BI || !BI->isConditional())
- return false;
- Cond = dyn_cast<ICmpInst>(BI->getCondition());
- if (!Cond)
- return false;
- Value *B1Op0 = Cond->getOperand(0);
- auto Cond1Op = Cond->getPredicate();
-
- if (B1Op0 != B2Op0)
- return false;
-
- if (Cond1Op == ICmpInst::ICMP_SGT || Cond1Op == ICmpInst::ICMP_SGE) {
- if (Cond2Op != ICmpInst::ICMP_SLT && Cond2Op != ICmpInst::ICMP_SLE)
- return false;
- } else if (Cond1Op == ICmpInst::ICMP_SLT || Cond1Op == ICmpInst::ICMP_SLE) {
- if (Cond2Op != ICmpInst::ICMP_SGT && Cond2Op != ICmpInst::ICMP_SGE)
- return false;
- } else if (Cond1Op == ICmpInst::ICMP_ULT || Cond1Op == ICmpInst::ICMP_ULE) {
- if (Cond2Op != ICmpInst::ICMP_UGT && Cond2Op != ICmpInst::ICMP_UGE)
- return false;
- } else if (Cond1Op == ICmpInst::ICMP_UGT || Cond1Op == ICmpInst::ICMP_UGE) {
- if (Cond2Op != ICmpInst::ICMP_ULT && Cond2Op != ICmpInst::ICMP_ULE)
- return false;
- } else {
- return false;
- }
-
- PassThroughInfo Info(Cond, BI, 0);
- PassThroughs.push_back(Info);
-
- return true;
-}
-
-// To avoid speculative hoisting certain computations out of
-// a basic block.
-bool BPFAdjustOptImpl::avoidSpeculation(Instruction &I) {
- if (auto *LdInst = dyn_cast<LoadInst>(&I)) {
- if (auto *GV = dyn_cast<GlobalVariable>(LdInst->getOperand(0))) {
- if (GV->hasAttribute(BPFCoreSharedInfo::AmaAttr) ||
- GV->hasAttribute(BPFCoreSharedInfo::TypeIdAttr))
- return false;
- }
- }
-
- if (!isa<LoadInst>(&I) && !isa<CallInst>(&I))
- return false;
-
- // For:
- // B1:
- // var = ...
- // ...
- // /* icmp may not be in the same block as var = ... */
- // comp1 = icmp <opcode> var, <const>;
- // if (comp1) goto B2 else B3;
- // B2:
- // ... var ...
- // change to:
- // B1:
- // var = ...
- // ...
- // /* icmp may not be in the same block as var = ... */
- // comp1 = icmp <opcode> var, <const>;
- // if (comp1) goto B2 else B3;
- // B2:
- // var = __builtin_bpf_passthrough(seq_num, var);
- // ... var ...
- bool isCandidate = false;
- SmallVector<PassThroughInfo, 4> Candidates;
- for (User *U : I.users()) {
- Instruction *Inst = dyn_cast<Instruction>(U);
- if (!Inst)
- continue;
-
- // May cover a little bit more than the
- // above pattern.
- if (auto *Icmp1 = dyn_cast<ICmpInst>(Inst)) {
- Value *Icmp1Op1 = Icmp1->getOperand(1);
- if (!isa<Constant>(Icmp1Op1))
- return false;
- isCandidate = true;
- continue;
- }
-
- // Ignore the use in the same basic block as the definition.
- if (Inst->getParent() == I.getParent())
- continue;
-
- // use in a different basic block, If there is a call or
- // load/store insn before this instruction in this basic
- // block. Most likely it cannot be hoisted out. Skip it.
- for (auto &I2 : *Inst->getParent()) {
- if (isa<CallInst>(&I2))
- return false;
- if (isa<LoadInst>(&I2) || isa<StoreInst>(&I2))
- return false;
- if (&I2 == Inst)
- break;
- }
-
- // It should be used in a GEP or a simple arithmetic like
- // ZEXT/SEXT which is used for GEP.
- if (Inst->getOpcode() == Instruction::ZExt ||
- Inst->getOpcode() == Instruction::SExt) {
- PassThroughInfo Info(&I, Inst, 0);
- Candidates.push_back(Info);
- } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
- // traverse GEP inst to find Use operand index
- unsigned i, e;
- for (i = 1, e = GI->getNumOperands(); i != e; ++i) {
- Value *V = GI->getOperand(i);
- if (V == &I)
- break;
- }
- if (i == e)
- continue;
-
- PassThroughInfo Info(&I, GI, i);
- Candidates.push_back(Info);
- }
- }
-
- if (!isCandidate || Candidates.empty())
- return false;
-
- llvm::append_range(PassThroughs, Candidates);
- return true;
-}
-
-void BPFAdjustOptImpl::adjustBasicBlock(BasicBlock &BB) {
- if (!DisableBPFserializeICMP && serializeICMPCrossBB(BB))
- return;
-}
-
-void BPFAdjustOptImpl::adjustInst(Instruction &I) {
- if (!DisableBPFserializeICMP && serializeICMPInBB(I))
- return;
- if (!DisableBPFavoidSpeculation && avoidSpeculation(I))
- return;
-}
-
-PreservedAnalyses BPFAdjustOptPass::run(Module &M, ModuleAnalysisManager &AM) {
- return BPFAdjustOptImpl(&M).run() ? PreservedAnalyses::none()
- : PreservedAnalyses::all();
-}
diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index 2b78ed7134c92..9672c57ae0238 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -54,7 +54,6 @@ class BPFCheckAndAdjustIR final : public ModulePass {
void checkIR(Module &M);
bool adjustIR(Module &M);
bool removePassThroughBuiltin(Module &M);
- bool removeCompareBuiltin(Module &M);
bool sinkMinMax(Module &M);
bool removeGEPBuiltins(Module &M);
bool insertASpaceCasts(Module &M);
@@ -132,46 +131,6 @@ bool BPFCheckAndAdjustIR::removePassThroughBuiltin(Module &M) {
return Changed;
}
-bool BPFCheckAndAdjustIR::removeCompareBuiltin(Module &M) {
- // Remove __builtin_bpf_compare()'s which are used to prevent
- // certain IR optimizations. Now major IR optimizations are done,
- // remove them.
- bool Changed = false;
- CallInst *ToBeDeleted = nullptr;
- for (Function &F : M)
- for (auto &BB : F)
- for (auto &I : BB) {
- if (ToBeDeleted) {
- ToBeDeleted->eraseFromParent();
- ToBeDeleted = nullptr;
- }
-
- auto *Call = dyn_cast<CallInst>(&I);
- if (!Call)
- continue;
- auto *GV = dyn_cast<GlobalValue>(Call->getCalledOperand());
- if (!GV)
- continue;
- if (!GV->getName().starts_with("llvm.bpf.compare"))
- continue;
-
- Changed = true;
- Value *Arg0 = Call->getArgOperand(0);
- Value *Arg1 = Call->getArgOperand(1);
- Value *Arg2 = Call->getArgOperand(2);
-
- auto OpVal = cast<ConstantInt>(Arg0)->getValue().getZExtValue();
- CmpInst::Predicate Opcode = (CmpInst::Predicate)OpVal;
-
- auto *ICmp = new ICmpInst(Opcode, Arg1, Arg2);
- ICmp->insertBefore(Call);
-
- Call->replaceAllUsesWith(ICmp);
- ToBeDeleted = Call;
- }
- return Changed;
-}
-
struct MinMaxSinkInfo {
ICmpInst *ICmp;
Value *Other;
@@ -532,12 +491,7 @@ bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) {
}
bool BPFCheckAndAdjustIR::adjustIR(Module &M) {
- bool Changed = removePassThroughBuiltin(M);
- Changed = removeCompareBuiltin(M) || Changed;
- Changed = sinkMinMax(M) || Changed;
- Changed = removeGEPBuiltins(M) || Changed;
- Changed = insertASpaceCasts(M) || Changed;
- return Changed;
+ return removePassThroughBuiltin(M);
}
bool BPFCheckAndAdjustIR::runOnModule(Module &M) {
diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
index 7d91fa8bb824c..efb4cb54f3878 100644
--- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp
+++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
@@ -137,10 +137,6 @@ void BPFTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) {
// SimplifyCFGPass(... .sinkCommonInsts(true))
FPM.addPass(BPFPreserveStaticOffsetPass(false));
});
- PB.registerPipelineEarlySimplificationEPCallback(
- [=](ModulePassManager &MPM, OptimizationLevel) {
- MPM.addPass(BPFAdjustOptPass());
- });
}
void BPFPassConfig::addIRPasses() {
diff --git a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
index 0ecff32a7525f..10389f667e9b7 100644
--- a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
+++ b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h
@@ -81,6 +81,9 @@ class BPFTTIImpl : public BasicTTIImplBase<BPFTTIImpl> {
return 5;
}
+ bool needsPreserveRangeInfoInVerification() const {
+ return true;
+ }
};
} // end namespace llvm
diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt
index eade4cacb7100..a432411193942 100644
--- a/llvm/lib/Target/BPF/CMakeLists.txt
+++ b/llvm/lib/Target/BPF/CMakeLists.txt
@@ -22,7 +22,6 @@ add_llvm_target(BPFCodeGen
GISel/BPFRegisterBankInfo.cpp
GISel/BPFLegalizerInfo.cpp
BPFAbstractMemberAccess.cpp
- BPFAdjustOpt.cpp
BPFAsmPrinter.cpp
BPFASpaceCastSimplifyPass.cpp
BPFCheckAndAdjustIR.cpp
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index b8664089c36ce..4fc928ebd5f2b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -13,6 +13,7 @@
#include "InstCombineInternal.h"
#include "llvm/Analysis/CmpInstAnalysis.h"
#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
@@ -1222,6 +1223,9 @@ static Value *foldAndOrOfICmpsWithConstEq(ICmpInst *Cmp0, ICmpInst *Cmp1,
Value *InstCombinerImpl::foldAndOrOfICmpsUsingRanges(ICmpInst *ICmp1,
ICmpInst *ICmp2,
bool IsAnd) {
+ if (TTI.needsPreserveRangeInfoInVerification())
+ return nullptr;
+
ICmpInst::Predicate Pred1, Pred2;
Value *V1, *V2;
const APInt *C1, *C2;
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 75883e0da214d..86567bcf1eee3 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -191,7 +191,7 @@ static bool pointerInvalidatedByLoop(MemorySSA *MSSA, MemoryUse *MU,
static bool pointerInvalidatedByBlock(BasicBlock &BB, MemorySSA &MSSA,
MemoryUse &MU);
/// Aggregates various functions for hoisting computations out of loop.
-static bool hoistArithmetics(Instruction &I, Loop &L,
+static bool hoistArithmetics(TargetTransformInfo *TTI, Instruction &I, Loop &L,
ICFLoopSafetyInfo &SafetyInfo,
MemorySSAUpdater &MSSAU, AssumptionCache *AC,
DominatorTree *DT);
@@ -467,9 +467,9 @@ bool LoopInvariantCodeMotion::runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI,
MSSAU, &SafetyInfo, Flags, ORE);
Flags.setIsSink(false);
if (Preheader)
- Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, AC, TLI, L,
- MSSAU, SE, &SafetyInfo, Flags, ORE, LoopNestMode,
- LicmAllowSpeculation);
+ Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, AC, TLI,
+ TTI, L, MSSAU, SE, &SafetyInfo, Flags, ORE,
+ LoopNestMode, LicmAllowSpeculation);
// Now that all loop invariants have been removed from the loop, promote any
// memory references to scalars that we can.
@@ -873,9 +873,9 @@ class ControlFlowHoister {
///
bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
DominatorTree *DT, AssumptionCache *AC,
- TargetLibraryInfo *TLI, Loop *CurLoop,
- MemorySSAUpdater &MSSAU, ScalarEvolution *SE,
- ICFLoopSafetyInfo *SafetyInfo,
+ TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
+ Loop *CurLoop, MemorySSAUpdater &MSSAU,
+ ScalarEvolution *SE, ICFLoopSafetyInfo *SafetyInfo,
SinkAndHoistLICMFlags &Flags,
OptimizationRemarkEmitter *ORE, bool LoopNestMode,
bool AllowSpeculation) {
@@ -987,7 +987,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
// Try to reassociate instructions so that part of computations can be
// done out of loop.
- if (hoistArithmetics(I, *CurLoop, *SafetyInfo, MSSAU, AC, DT)) {
+ if (hoistArithmetics(TTI, I, *CurLoop, *SafetyInfo, MSSAU, AC, DT)) {
Changed = true;
continue;
}
@@ -2406,8 +2406,11 @@ bool pointerInvalidatedByBlock(BasicBlock &BB, MemorySSA &MSSA, MemoryUse &MU) {
/// Try to simplify things like (A < INV_1 AND icmp A < INV_2) into (A <
/// min(INV_1, INV_2)), if INV_1 and INV_2 are both loop invariants and their
/// minimun can be computed outside of loop, and X is not a loop-invariant.
-static bool hoistMinMax(Instruction &I, Loop &L, ICFLoopSafetyInfo &SafetyInfo,
- MemorySSAUpdater &MSSAU) {
+static bool hoistMinMax(TargetTransformInfo *TTI, Instruction &I, Loop &L,
+ ICFLoopSafetyInfo &SafetyInfo, MemorySSAUpdater &MSSAU) {
+ if (TTI->needsPreserveRangeInfoInVerification())
+ return false;
+
bool Inverse = false;
using namespace PatternMatch;
Value *Cond1, *Cond2;
@@ -2779,14 +2782,14 @@ static bool hoistMulAddAssociation(Instruction &I, Loop &L,
return true;
}
-static bool hoistArithmetics(Instruction &I, Loop &L,
+static bool hoistArithmetics(TargetTransformInfo *TTI, Instruction &I, Loop &L,
ICFLoopSafetyInfo &SafetyInfo,
MemorySSAUpdater &MSSAU, AssumptionCache *AC,
DominatorTree *DT) {
// Optimize complex patterns, such as (x < INV1 && x < INV2), turning them
// into (x < min(INV1, INV2)), and hoisting the invariant part of this
// expression out of the loop.
- if (hoistMinMax(I, L, SafetyInfo, MSSAU)) {
+ if (hoistMinMax(TTI, I, L, SafetyInfo, MSSAU)) {
++NumHoisted;
++NumMinMaxHoisted;
return true;
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index c52c4dc0b8a51..31f0ef2a06226 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -3477,6 +3477,9 @@ static bool FoldCondBranchOnValueKnownInPredecessor(BranchInst *BI,
/// see if we can eliminate it.
static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
DomTreeUpdater *DTU, const DataLayout &DL) {
+ if (TTI.needsPreserveRangeInfoInVerification())
+ return false;
+
// Ok, this is a two entry PHI node. Check to see if this is a simple "if
// statement", which has a very simple dominance structure. Basically, we
// are trying to find the condition that is being branched on, which
diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll
index be8675f084302..99ec657e3ae1c 100644
--- a/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll
+++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll
@@ -1,11 +1,7 @@
; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s
+; RUN: llc %t1 -o - | FileCheck %s
; RUN: opt -passes='default<O2>' -mtriple=bpf-pc-linux %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s
-; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s
-; RUN: opt -passes='default<O2>' -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s
+; RUN: llc %t1 -o - | FileCheck %s
;
; Source:
; int foo();
@@ -38,12 +34,6 @@ entry:
; CHECK: if [[REG2]] s> [[REG1]] goto
; CHECK: if [[REG1]] s> 7 goto
-; CHECK-DISABLE: [[REG1:r[0-9]+]] += -8
-; CHECK-DISABLE: [[REG1]] <<= 32
-; CHECK-DISABLE: [[REG1]] >>= 32
-; CHECK-DISABLE: [[REG2:r[0-9]+]] = 4294967289
-; CHECK-DISABLE: if [[REG2]] > [[REG1]] goto
-
lor.lhs.false: ; preds = %entry
%1 = load i32, ptr %ret, align 4, !tbaa !2
%cmp1 = icmp sgt i32 %1, 7
diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll
index cfc9e886653a2..37bc04f58d433 100644
--- a/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll
+++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll
@@ -1,7 +1,5 @@
; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s
-; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s
+; RUN: llc %t1 -o - | FileCheck %s
;
; Source:
; int foo();
@@ -36,12 +34,6 @@ entry:
; CHECK: if [[REG2]] s> [[REG1]] goto
; CHECK: if [[REG1]] s> 7 goto
-; CHECK-DISABLE: [[REG1:r[0-9]+]] += -8
-; CHECK-DISABLE: [[REG1]] <<= 32
-; CHECK-DISABLE: [[REG1]] >>= 32
-; CHECK-DISABLE: [[REG2:r[0-9]+]] = 4294967289
-; CHECK-DISABLE: if [[REG2]] > [[REG1]] goto
-
if.then: ; preds = %entry
store i32 0, ptr %retval, align 4
store i32 1, ptr %cleanup.dest.slot, align 4
diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll
index 5b81a01e51835..a12d370839a82 100644
--- a/llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll
+++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll
@@ -1,7 +1,7 @@
; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK,CHECK-V1 %s
+; RUN: llc %t1 -o - | FileCheck %s
; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1
-; RUN: llc %t1 -mcpu=v3 -o - | FileCheck -check-prefixes=CHECK,CHECK-V3 %s
+; RUN: llc %t1 -mcpu=v3 -o - | FileCheck %s
;
; Source:
; int test1(unsigned long a) {
@@ -40,8 +40,8 @@ return: ; preds = %if.end, %if.then
}
; CHECK-LABEL: test1
-; CHECK-V1: if r[[#]] > r[[#]] goto
-; CHECK-V3: if w[[#]] < 4 goto
+; CHECK: r[[#]] &= r[[#]]
+; CHECK: if r[[#]] == 0 goto
; Function Attrs: nounwind
define dso_local i32 @test2(i64 %a) #0 {
@@ -68,8 +68,8 @@ return: ; preds = %if.end, %if.then
}
; CHECK-LABEL: test2
-; CHECK-V1: if r[[#]] > r[[#]] goto
-; CHECK-V3: if w[[#]] < 4 goto
+; CHECK: r[[#]] &= r[[#]]
+; CHECK: if r[[#]] == 0 goto
attributes #0 = { nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll
index 0c2b7cb9db9f3..9f32ded4e92d2 100644
--- a/llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll
+++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll
@@ -1,7 +1,7 @@
; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK,CHECK-V1 %s
+; RUN: llc %t1 -o - | FileCheck %s
; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1
-; RUN: llc %t1 -mcpu=v3 -o - | FileCheck -check-prefixes=CHECK,CHECK-V3 %s
+; RUN: llc %t1 -mcpu=v3 -o - | FileCheck %s
;
; Source:
; int test1(unsigned long a) {
@@ -40,8 +40,8 @@ return: ; preds = %if.end, %if.then
}
; CHECK-LABEL: test1
-; CHECK-V1: if r[[#]] > 3 goto
-; CHECK-V3: if w[[#]] > 3 goto
+; CHECK: r[[#]] &= r[[#]]
+; CHECK: if r[[#]] == 0 goto
; Function Attrs: nounwind
define dso_local i32 @test2(i64 %a) #0 {
@@ -68,8 +68,8 @@ return: ; preds = %if.end, %if.then
}
; CHECK-LABEL: test2
-; CHECK-V1: if r[[#]] > 3 goto
-; CHECK-V3: if w[[#]] > 3 goto
+; CHECK: r[[#]] &= r[[#]]
+; CHECK: if r[[#]] == 0 goto
attributes #0 = { nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
diff --git a/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll b/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll
index 5a38f5ec19695..cd6c80abffc41 100644
--- a/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll
+++ b/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll
@@ -1,7 +1,5 @@
; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s
-; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s
+; RUN: llc %t1 -o - | FileCheck %s
;
; Source:
; unsigned long foo();
@@ -39,22 +37,15 @@ if.end: ; preds = %if.then, %entry
call void @llvm.lifetime.end.p0(i64 8, ptr %ret) #3
ret ptr %3
}
-; CHECK-COMMON: [[REG6:r[0-9]+]] = r1
-; CHECK-COMMON: call foo
+; CHECK: [[REG6:r[0-9]+]] = r1
+; CHECK: call foo
; CHECK: if r0 > 7 goto [[LABEL:.*]]
; CHECK: [[REG6]] += r0
; CHECK: [[LABEL]]:
; CHECK: r0 = [[REG6]]
-; CHECK-DISABLE: [[REG1:r[0-9]+]] = 8
-; CHECK-DISABLE: if [[REG1]] > r0 goto [[LABEL:.*]]
-; CHECK-DISABLE: r0 = 0
-; CHECK-DISABLE: [[LABEL]]:
-; CHECK-DISABLE: [[REG6]] += r0
-; CHECK-DISABLE: r0 = [[REG6]]
-
-; CHECK-COMMON: exit
+; CHECK: exit
; Function Attrs: argmemonly nounwind willreturn
declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
diff --git a/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll b/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll
index 2b841865d7243..10e35a203bad5 100644
--- a/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll
+++ b/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll
@@ -1,7 +1,5 @@
; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s
-; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s
+; RUN: llc %t1 -o - | FileCheck %s
;
; Source:
; unsigned foo();
@@ -41,8 +39,8 @@ if.end: ; preds = %if.then, %entry
ret ptr %3
}
-; CHECK-COMMON: [[REG6:r[0-9]+]] = r1
-; CHECK-COMMON: call foo
+; CHECK: [[REG6:r[0-9]+]] = r1
+; CHECK: call foo
; CHECK: r0 <<= 32
; CHECK: r0 >>= 32
@@ -51,19 +49,7 @@ if.end: ; preds = %if.then, %entry
; CHECK: [[LABEL]]:
; CHECK: r0 = [[REG6]]
-; CHECK-DISABLE: [[REG1:r[0-9]+]] = r0
-; CHECK-DISABLE: [[REG1]] <<= 32
-; CHECK-DISABLE: [[REG1]] >>= 32
-; CHECK-DISABLE: [[REG2:r[0-9]+]] = 8
-; CHECK-DISABLE: if [[REG2]] > [[REG1]] goto [[LABEL:.*]]
-; CHECK-DISABLE: r0 = 0
-; CHECK-DISABLE: [[LABEL]]:
-; CHECK-DISABLE: r0 <<= 32
-; CHECK-DISABLE: r0 >>= 32
-; CHECK-DISABLE: [[REG6]] += r0
-; CHECK-DISABLE: r0 = [[REG6]]
-
-; CHECK-COMMON: exit
+; CHECK: exit
; Function Attrs: argmemonly nounwind willreturn
declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1
More information about the llvm-commits
mailing list