[llvm] Add Promote constant pass to ARM (PR #196844)
via llvm-commits
llvm-commits at lists.llvm.org
Sun May 10 14:34:39 PDT 2026
https://github.com/LumioseSil updated https://github.com/llvm/llvm-project/pull/196844
>From d4210ba0e13c3aec48b675acab34cf2b60899a6f Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Sun, 10 May 2026 17:17:30 -0400
Subject: [PATCH] Add Promote constant pass to ARM
This helps with global merge.
---
llvm/lib/Target/ARM/ARM.h | 3 +
llvm/lib/Target/ARM/ARMPromoteConstant.cpp | 583 ++++++++++++++++++
llvm/lib/Target/ARM/ARMTargetMachine.cpp | 14 +-
llvm/lib/Target/ARM/CMakeLists.txt | 1 +
.../ARM/GlobalISel/arm-irtranslator.ll | 4 +-
llvm/test/CodeGen/ARM/O3-pipeline.ll | 5 +
llvm/test/CodeGen/ARM/arm-promote-const.ll | 181 ++++++
.../gn/secondary/llvm/lib/Target/ARM/BUILD.gn | 1 +
8 files changed, 788 insertions(+), 4 deletions(-)
create mode 100644 llvm/lib/Target/ARM/ARMPromoteConstant.cpp
create mode 100644 llvm/test/CodeGen/ARM/arm-promote-const.ll
diff --git a/llvm/lib/Target/ARM/ARM.h b/llvm/lib/Target/ARM/ARM.h
index 08caef8419d12..84a608b262877 100644
--- a/llvm/lib/Target/ARM/ARM.h
+++ b/llvm/lib/Target/ARM/ARM.h
@@ -30,8 +30,10 @@ class FunctionPass;
class InstructionSelector;
class MCInst;
class MachineInstr;
+class ModulePass;
class PassRegistry;
+ModulePass *createARMPromoteConstantPass();
Pass *createMVETailPredicationPass();
FunctionPass *createARMLowOverheadLoopsPass();
FunctionPass *createARMBlockPlacementPass();
@@ -63,6 +65,7 @@ void LowerARMMachineInstrToMCInst(const MachineInstr *MI, MCInst &OutMI,
ARMAsmPrinter &AP);
void initializeARMAsmPrinterPass(PassRegistry &);
+void initializeARMPromoteConstantPass(PassRegistry &);
void initializeARMBlockPlacementPass(PassRegistry &);
void initializeARMBranchTargetsPass(PassRegistry &);
void initializeARMConstantIslandsPass(PassRegistry &);
diff --git a/llvm/lib/Target/ARM/ARMPromoteConstant.cpp b/llvm/lib/Target/ARM/ARMPromoteConstant.cpp
new file mode 100644
index 0000000000000..e66f83e72c296
--- /dev/null
+++ b/llvm/lib/Target/ARM/ARMPromoteConstant.cpp
@@ -0,0 +1,583 @@
+//==- ARMPromoteConstant.cpp - Promote constant to global for ARM
+//----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the ARMPromoteConstant pass which promotes constants
+// to global variables when this is likely to be more efficient. Currently only
+// types related to constant vector (i.e., constant vector, array of constant
+// vectors, constant structure with a constant vector field, etc.) are promoted
+// to global variables. Constant vectors are likely to be lowered in target
+// constant pool during instruction selection already; therefore, the access
+// will remain the same (memory load), but the structure types are not split
+// into different constant pool accesses for each field. A bonus side effect is
+// that created globals may be merged by the global merge pass.
+//
+// Ported from AArch64PromoteConstant for ARM NEON / vector aggregates.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ARM.h"
+#include "ARMSubtarget.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <utility>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "arm-promote-const"
+
+// Stress testing mode - disable heuristics.
+static cl::opt<bool> Stress("arm-stress-promote-const", cl::Hidden,
+ cl::desc("Promote all vector constants"));
+
+STATISTIC(NumPromoted, "Number of promoted constants");
+STATISTIC(NumPromotedUses, "Number of promoted constants uses");
+
+//===----------------------------------------------------------------------===//
+// ARMPromoteConstant
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+/// Promotes interesting constant into global variables.
+/// See AArch64PromoteConstant for the motivating example.
+class ARMPromoteConstant : public ModulePass {
+public:
+ struct PromotedConstant {
+ bool ShouldConvert = false;
+ GlobalVariable *GV = nullptr;
+ };
+ using PromotionCacheTy = SmallDenseMap<Constant *, PromotedConstant, 16>;
+
+ struct UpdateRecord {
+ Constant *C;
+ Instruction *User;
+ unsigned Op;
+
+ UpdateRecord(Constant *C, Instruction *User, unsigned Op)
+ : C(C), User(User), Op(Op) {}
+ };
+
+ static char ID;
+
+ ARMPromoteConstant() : ModulePass(ID) {}
+
+ StringRef getPassName() const override { return "ARM Promote Constant"; }
+
+ /// Iterate over the functions and promote the interesting constants into
+ /// global variables with module scope.
+ bool runOnModule(Module &M) override {
+ LLVM_DEBUG(dbgs() << getPassName() << '\n');
+ if (skipModule(M))
+ return false;
+ const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
+ const TargetMachine &TM = TPC.getTM<TargetMachine>();
+ bool Changed = false;
+ PromotionCacheTy PromotionCache;
+ for (auto &MF : M) {
+ const ARMSubtarget &ST = TM.getSubtarget<ARMSubtarget>(MF);
+ Changed |= runOnFunction(MF, ST, PromotionCache);
+ }
+ return Changed;
+ }
+
+private:
+ /// Look for interesting constants used within the given function.
+ /// Promote them into global variables, load these global variables within
+ /// the related function, so that the number of inserted load is minimal.
+ bool runOnFunction(Function &F, const ARMSubtarget &ST,
+ PromotionCacheTy &PromotionCache);
+
+ // This transformation requires dominator info
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ AU.addRequired<TargetPassConfig>();
+ AU.addRequired<DominatorTreeWrapperPass>();
+ AU.addPreserved<DominatorTreeWrapperPass>();
+ }
+
+ /// Type to store a list of Uses.
+ using Uses = SmallVector<std::pair<Instruction *, unsigned>, 4>;
+ /// Map an insertion point to all the uses it dominates.
+ using InsertionPoints = DenseMap<Instruction *, Uses>;
+
+ /// Find the closest point that dominates the given Use.
+ Instruction *findInsertionPoint(Instruction &User, unsigned OpNo);
+
+ /// Check if the given insertion point is dominated by an existing
+ /// insertion point.
+ /// If true, the given use is added to the list of dominated uses for
+ /// the related existing point.
+ /// \param NewPt the insertion point to be checked
+ /// \param User the user of the constant
+ /// \param OpNo the operand number of the use
+ /// \param InsertPts existing insertion points
+ /// \pre NewPt and all instruction in InsertPts belong to the same function
+ /// \return true if one of the insertion point in InsertPts dominates NewPt,
+ /// false otherwise
+ bool isDominated(Instruction *NewPt, Instruction *User, unsigned OpNo,
+ InsertionPoints &InsertPts);
+
+ /// Check if the given insertion point can be merged with an existing
+ /// insertion point in a common dominator.
+ /// If true, the given use is added to the list of the created insertion
+ /// point.
+ /// \param NewPt the insertion point to be checked
+ /// \param User the user of the constant
+ /// \param OpNo the operand number of the use
+ /// \param InsertPts existing insertion points
+ /// \pre NewPt and all instruction in InsertPts belong to the same function
+ /// \pre isDominated returns false for the exact same parameters.
+ /// \return true if it exists an insertion point in InsertPts that could
+ /// have been merged with NewPt in a common dominator,
+ /// false otherwise
+ bool tryAndMerge(Instruction *NewPt, Instruction *User, unsigned OpNo,
+ InsertionPoints &InsertPts);
+
+ /// Compute the minimal insertion points to dominates all the interesting
+ /// uses of value.
+ /// Insertion points are group per function and each insertion point
+ /// contains a list of all the uses it dominates within the related function
+ /// \param User the user of the constant
+ /// \param OpNo the operand number of the constant
+ /// \param[out] InsertPts output storage of the analysis
+ void computeInsertionPoint(Instruction *User, unsigned OpNo,
+ InsertionPoints &InsertPts);
+
+ /// Insert a definition of a new global variable at each point contained in
+ /// InsPtsPerFunc and update the related uses (also contained in
+ /// InsPtsPerFunc).
+ void insertDefinitions(Function &F, GlobalVariable &GV,
+ InsertionPoints &InsertPts);
+
+ /// Do the constant promotion indicated by the Updates records, keeping track
+ /// of globals in PromotionCache.
+ void promoteConstants(Function &F, SmallVectorImpl<UpdateRecord> &Updates,
+ PromotionCacheTy &PromotionCache);
+
+ /// Transfer the list of dominated uses of IPI to NewPt in InsertPts.
+ /// Append Use to this list and delete the entry of IPI in InsertPts.
+ static void appendAndTransferDominatedUses(Instruction *NewPt,
+ Instruction *User, unsigned OpNo,
+ InsertionPoints::iterator &IPI,
+ InsertionPoints &InsertPts) {
+ // Record the dominated use.
+ IPI->second.emplace_back(User, OpNo);
+ // Transfer the dominated uses of IPI to NewPt
+ // Inserting into the DenseMap may invalidate existing iterator.
+ // Keep a copy of the key to find the iterator to erase. Keep a copy of the
+ // value so that we don't have to dereference IPI->second.
+ Instruction *OldInstr = IPI->first;
+ Uses OldUses = std::move(IPI->second);
+ InsertPts[NewPt] = std::move(OldUses);
+ // Erase IPI.
+ InsertPts.erase(OldInstr);
+ }
+};
+
+} // end anonymous namespace
+
+char ARMPromoteConstant::ID = 0;
+
+INITIALIZE_PASS_BEGIN(ARMPromoteConstant, "arm-promote-const",
+ "ARM Promote Constant Pass", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_END(ARMPromoteConstant, "arm-promote-const",
+ "ARM Promote Constant Pass", false, false)
+
+ModulePass *llvm::createARMPromoteConstantPass() {
+ return new ARMPromoteConstant();
+}
+
+/// Check if the given type uses a vector type.
+static bool isConstantUsingVectorTy(const Type *CstTy) {
+ if (CstTy->isVectorTy())
+ return true;
+ if (CstTy->isStructTy()) {
+ for (unsigned EltIdx = 0, EndEltIdx = CstTy->getStructNumElements();
+ EltIdx < EndEltIdx; ++EltIdx)
+ if (isConstantUsingVectorTy(CstTy->getStructElementType(EltIdx)))
+ return true;
+ } else if (CstTy->isArrayTy())
+ return isConstantUsingVectorTy(CstTy->getArrayElementType());
+ return false;
+}
+
+// Returns true if \p C contains only ConstantData leaves and no global values,
+// block addresses or constant expressions. Traverses ConstantAggregates.
+static bool containsOnlyConstantData(const Constant *C) {
+ if (isa<ConstantData>(C))
+ return true;
+
+ if (isa<GlobalValue>(C) || isa<BlockAddress>(C) || isa<ConstantExpr>(C))
+ return false;
+
+ return all_of(C->operands(), [](const Use &U) {
+ return containsOnlyConstantData(cast<Constant>(&U));
+ });
+}
+
+/// Check if the given use (Instruction + OpIdx) of Cst should be converted into
+/// a load of a global variable initialized with Cst.
+/// A use should be converted if it is legal to do so.
+/// For instance, it is not legal to turn the mask operand of a shuffle vector
+/// into a load of a global variable.
+static bool shouldConvertUse(const Constant *Cst, const Instruction *Instr,
+ unsigned OpIdx) {
+ // shufflevector instruction expects a const for the mask argument, i.e., the
+ // third argument. Do not promote this use in that case.
+ if (isa<const ShuffleVectorInst>(Instr) && OpIdx == 2)
+ return false;
+
+ // extractvalue instruction expects a const idx.
+ if (isa<const ExtractValueInst>(Instr) && OpIdx > 0)
+ return false;
+
+ // extractvalue instruction expects a const idx.
+ if (isa<const InsertValueInst>(Instr) && OpIdx > 1)
+ return false;
+
+ if (isa<const AllocaInst>(Instr) && OpIdx > 0)
+ return false;
+
+ // Alignment argument must be constant.
+ if (isa<const LoadInst>(Instr) && OpIdx > 0)
+ return false;
+
+ // Alignment argument must be constant.
+ if (isa<const StoreInst>(Instr) && OpIdx > 1)
+ return false;
+
+ // Index must be constant.
+ if (isa<const GetElementPtrInst>(Instr) && OpIdx > 0)
+ return false;
+
+ // Personality function and filters must be constant.
+ // Give up on that instruction.
+ if (isa<const LandingPadInst>(Instr))
+ return false;
+
+ // Switch instruction expects constants to compare to.
+ if (isa<const SwitchInst>(Instr))
+ return false;
+
+ // Expected address must be a constant.
+ if (isa<const IndirectBrInst>(Instr))
+ return false;
+
+ // Do not mess with intrinsics.
+ if (isa<const IntrinsicInst>(Instr))
+ return false;
+
+ // Do not mess with inline asm.
+ const CallInst *CI = dyn_cast<const CallInst>(Instr);
+ return !(CI && CI->isInlineAsm());
+}
+
+/// Check if the given Cst should be converted into
+/// a load of a global variable initialized with Cst.
+/// A constant should be converted if it is likely that the materialization of
+/// the constant will be tricky. Thus, we give up on zero or undef values.
+///
+/// \todo Currently, accept only vector related types.
+/// Also we give up on all simple vector type to keep the existing
+/// behavior. Otherwise, we should push here all the check of the lowering of
+/// BUILD_VECTOR. By giving up, we lose the potential benefit of merging
+/// constant via global merge and the fact that the same constant is stored
+/// only once with this method (versus, as many function that uses the constant
+/// for the regular approach, even for float).
+/// Again, the simplest solution would be to promote every
+/// constant and rematerialize them when they are actually cheap to create.
+static bool shouldConvertImpl(const Constant *Cst) {
+ if (isa<const UndefValue>(Cst))
+ return false;
+
+ // FIXME: In some cases, it may be interesting to promote in memory
+ // a zero initialized constant.
+ // E.g., when the type of Cst require more instructions than the
+ // adrp/add/load sequence or when this sequence can be shared by several
+ // instances of Cst.
+ // Ideally, we could promote this into a global and rematerialize the constant
+ // when it was a bad idea.
+ if (Cst->isNullValue())
+ return false;
+
+ // Globals cannot be or contain scalable vectors.
+ if (Cst->getType()->isScalableTy())
+ return false;
+
+ if (Stress)
+ return true;
+
+ // FIXME: see function \todo
+ if (Cst->getType()->isVectorTy())
+ return false;
+ return isConstantUsingVectorTy(Cst->getType());
+}
+
+static bool
+shouldConvert(Constant &C,
+ ARMPromoteConstant::PromotionCacheTy &PromotionCache) {
+ auto Converted = PromotionCache.insert(
+ std::make_pair(&C, ARMPromoteConstant::PromotedConstant()));
+ if (Converted.second)
+ Converted.first->second.ShouldConvert = shouldConvertImpl(&C);
+ return Converted.first->second.ShouldConvert;
+}
+
+Instruction *ARMPromoteConstant::findInsertionPoint(Instruction &User,
+ unsigned OpNo) {
+ // If this user is a phi, the insertion point is in the related
+ // incoming basic block.
+ if (PHINode *PhiInst = dyn_cast<PHINode>(&User))
+ return PhiInst->getIncomingBlock(OpNo)->getTerminator();
+
+ return &User;
+}
+
+bool ARMPromoteConstant::isDominated(Instruction *NewPt, Instruction *User,
+ unsigned OpNo,
+ InsertionPoints &InsertPts) {
+ DominatorTree &DT =
+ getAnalysis<DominatorTreeWrapperPass>(*NewPt->getParent()->getParent())
+ .getDomTree();
+
+ // Traverse all the existing insertion points and check if one is dominating
+ // NewPt. If it is, remember that.
+ for (auto &IPI : InsertPts) {
+ if (NewPt == IPI.first || DT.dominates(IPI.first, NewPt) ||
+ // When IPI.first is a terminator instruction, DT may think that
+ // the result is defined on the edge.
+ // Here we are testing the insertion point, not the definition.
+ (IPI.first->getParent() != NewPt->getParent() &&
+ DT.dominates(IPI.first->getParent(), NewPt->getParent()))) {
+ // No need to insert this point. Just record the dominated use.
+ LLVM_DEBUG(dbgs() << "Insertion point dominated by:\n");
+ LLVM_DEBUG(IPI.first->print(dbgs()));
+ LLVM_DEBUG(dbgs() << '\n');
+ IPI.second.emplace_back(User, OpNo);
+ return true;
+ }
+ }
+ return false;
+}
+
+bool ARMPromoteConstant::tryAndMerge(Instruction *NewPt, Instruction *User,
+ unsigned OpNo,
+ InsertionPoints &InsertPts) {
+ DominatorTree &DT =
+ getAnalysis<DominatorTreeWrapperPass>(*NewPt->getParent()->getParent())
+ .getDomTree();
+ BasicBlock *NewBB = NewPt->getParent();
+
+ // Traverse all the existing insertion point and check if one is dominated by
+ // NewPt and thus useless or can be combined with NewPt into a common
+ // dominator.
+ for (InsertionPoints::iterator IPI = InsertPts.begin(),
+ EndIPI = InsertPts.end();
+ IPI != EndIPI; ++IPI) {
+ BasicBlock *CurBB = IPI->first->getParent();
+ if (NewBB == CurBB) {
+ // Instructions are in the same block.
+ // By construction, NewPt is dominating the other.
+ // Indeed, isDominated returned false with the exact same arguments.
+ LLVM_DEBUG(dbgs() << "Merge insertion point with:\n");
+ LLVM_DEBUG(IPI->first->print(dbgs()));
+ LLVM_DEBUG(dbgs() << "\nat considered insertion point.\n");
+ appendAndTransferDominatedUses(NewPt, User, OpNo, IPI, InsertPts);
+ return true;
+ }
+
+ // Look for a common dominator
+ BasicBlock *CommonDominator = DT.findNearestCommonDominator(NewBB, CurBB);
+ // If none exists, we cannot merge these two points.
+ if (!CommonDominator)
+ continue;
+
+ if (CommonDominator != NewBB) {
+ // By construction, the CommonDominator cannot be CurBB.
+ assert(CommonDominator != CurBB &&
+ "Instruction has not been rejected during isDominated check!");
+ // Take the last instruction of the CommonDominator as insertion point
+ NewPt = CommonDominator->getTerminator();
+ }
+ // else, CommonDominator is the block of NewBB, hence NewBB is the last
+ // possible insertion point in that block.
+ LLVM_DEBUG(dbgs() << "Merge insertion point with:\n");
+ LLVM_DEBUG(IPI->first->print(dbgs()));
+ LLVM_DEBUG(dbgs() << '\n');
+ LLVM_DEBUG(NewPt->print(dbgs()));
+ LLVM_DEBUG(dbgs() << '\n');
+ appendAndTransferDominatedUses(NewPt, User, OpNo, IPI, InsertPts);
+ return true;
+ }
+ return false;
+}
+
+void ARMPromoteConstant::computeInsertionPoint(Instruction *User, unsigned OpNo,
+ InsertionPoints &InsertPts) {
+ LLVM_DEBUG(dbgs() << "Considered use, opidx " << OpNo << ":\n");
+ LLVM_DEBUG(User->print(dbgs()));
+ LLVM_DEBUG(dbgs() << '\n');
+
+ Instruction *InsertionPoint = findInsertionPoint(*User, OpNo);
+
+ LLVM_DEBUG(dbgs() << "Considered insertion point:\n");
+ LLVM_DEBUG(InsertionPoint->print(dbgs()));
+ LLVM_DEBUG(dbgs() << '\n');
+
+ if (isDominated(InsertionPoint, User, OpNo, InsertPts))
+ return;
+ // This insertion point is useful, check if we can merge some insertion
+ // point in a common dominator or if NewPt dominates an existing one.
+ if (tryAndMerge(InsertionPoint, User, OpNo, InsertPts))
+ return;
+
+ LLVM_DEBUG(dbgs() << "Keep considered insertion point\n");
+
+ // It is definitely useful by its own
+ InsertPts[InsertionPoint].emplace_back(User, OpNo);
+}
+
+static void ensurePromotedGV(Function &F, Constant &C,
+ ARMPromoteConstant::PromotedConstant &PC) {
+ assert(PC.ShouldConvert &&
+ "Expected that we should convert this to a global");
+ if (PC.GV)
+ return;
+ PC.GV = new GlobalVariable(
+ *F.getParent(), C.getType(), true, GlobalValue::InternalLinkage, nullptr,
+ "_PromotedConst", nullptr, GlobalVariable::NotThreadLocal);
+ PC.GV->setInitializer(&C);
+ LLVM_DEBUG(dbgs() << "Global replacement: ");
+ LLVM_DEBUG(PC.GV->print(dbgs()));
+ LLVM_DEBUG(dbgs() << '\n');
+ ++NumPromoted;
+}
+
+void ARMPromoteConstant::insertDefinitions(Function &F,
+ GlobalVariable &PromotedGV,
+ InsertionPoints &InsertPts) {
+#ifndef NDEBUG
+ // Do more checking for debug purposes.
+ DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>(F).getDomTree();
+#endif
+ assert(!InsertPts.empty() && "Empty uses does not need a definition");
+
+ for (const auto &IPI : InsertPts) {
+ // Create the load of the global variable.
+ IRBuilder<> Builder(IPI.first);
+ LoadInst *LoadedCst =
+ Builder.CreateLoad(PromotedGV.getValueType(), &PromotedGV);
+ LLVM_DEBUG(dbgs() << "**********\n");
+ LLVM_DEBUG(dbgs() << "New def: ");
+ LLVM_DEBUG(LoadedCst->print(dbgs()));
+ LLVM_DEBUG(dbgs() << '\n');
+
+ // Update the dominated uses.
+ for (auto Use : IPI.second) {
+#ifndef NDEBUG
+ assert(
+ DT.dominates(LoadedCst, findInsertionPoint(*Use.first, Use.second)) &&
+ "Inserted definition does not dominate all its uses!");
+#endif
+ LLVM_DEBUG({
+ dbgs() << "Use to update " << Use.second << ":";
+ Use.first->print(dbgs());
+ dbgs() << '\n';
+ });
+ Use.first->setOperand(Use.second, LoadedCst);
+ ++NumPromotedUses;
+ }
+ }
+}
+
+void ARMPromoteConstant::promoteConstants(
+ Function &F, SmallVectorImpl<UpdateRecord> &Updates,
+ PromotionCacheTy &PromotionCache) {
+ // Promote the constants.
+ for (auto U = Updates.begin(), E = Updates.end(); U != E;) {
+ LLVM_DEBUG(dbgs() << "** Compute insertion points **\n");
+ auto First = U;
+ Constant *C = First->C;
+ InsertionPoints InsertPts;
+ do {
+ computeInsertionPoint(U->User, U->Op, InsertPts);
+ } while (++U != E && U->C == C);
+
+ auto &Promotion = PromotionCache[C];
+ ensurePromotedGV(F, *C, Promotion);
+ insertDefinitions(F, *Promotion.GV, InsertPts);
+ }
+}
+
+bool ARMPromoteConstant::runOnFunction(Function &F, const ARMSubtarget &ST,
+ PromotionCacheTy &PromotionCache) {
+ if (!Stress && !ST.hasNEON() && !ST.hasMVEIntegerOps() &&
+ !ST.hasMVEFloatOps())
+ return false;
+
+ // Look for instructions using constant vector. Promote that constant to a
+ // global variable. Create as few loads of this variable as possible and
+ // update the uses accordingly.
+ SmallVector<UpdateRecord, 64> Updates;
+ for (Instruction &I : instructions(&F)) {
+ // Traverse the operand, looking for constant vectors. Replace them by a
+ // load of a global variable of constant vector type.
+ for (Use &U : I.operands()) {
+ Constant *Cst = dyn_cast<Constant>(U);
+ // There is no point in promoting global values as they are already
+ // global. Do not promote constants containing constant expression, global
+ // values or blockaddresses either, as they may require some code
+ // expansion.
+ if (!Cst || isa<GlobalValue>(Cst) || !containsOnlyConstantData(Cst))
+ continue;
+
+ // Check if this constant is worth promoting.
+ if (!shouldConvert(*Cst, PromotionCache))
+ continue;
+
+ // Check if this use should be promoted.
+ unsigned OpNo = &U - I.op_begin();
+ if (!shouldConvertUse(Cst, &I, OpNo))
+ continue;
+
+ Updates.emplace_back(Cst, &I, OpNo);
+ }
+ }
+
+ if (Updates.empty())
+ return false;
+
+ promoteConstants(F, Updates, PromotionCache);
+ return true;
+}
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index 0de6f3d16eff4..772730fea5e9f 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -77,8 +77,13 @@ EnableARMLoadStoreOpt("arm-load-store-opt", cl::Hidden,
// FIXME: Unify control over GlobalMerge.
static cl::opt<cl::boolOrDefault>
-EnableGlobalMerge("arm-global-merge", cl::Hidden,
- cl::desc("Enable the global merge pass"));
+ EnableGlobalMerge("arm-global-merge", cl::Hidden,
+ cl::desc("Enable the global merge pass"));
+
+static cl::opt<bool>
+ EnablePromoteConstant("arm-enable-promote-const",
+ cl::desc("Enable the promote constant pass"),
+ cl::init(true), cl::Hidden);
namespace llvm {
void initializeARMExecutionDomainFixPass(PassRegistry&);
@@ -100,6 +105,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeARMTarget() {
initializeARMBranchTargetsPass(Registry);
initializeARMConstantIslandsPass(Registry);
initializeARMExecutionDomainFixPass(Registry);
+ initializeARMPromoteConstantPass(Registry);
initializeARMExpandPseudoPass(Registry);
initializeThumb2SizeReducePass(Registry);
initializeMVEVPTBlockPass(Registry);
@@ -403,6 +409,10 @@ void ARMPassConfig::addCodeGenPrepare() {
}
bool ARMPassConfig::addPreISel() {
+ // Run promote constant before global merge, so that the promoted constants
+ // get a chance to be merged (matches AArch64 pipeline ordering).
+ if (TM->getOptLevel() != CodeGenOptLevel::None && EnablePromoteConstant)
+ addPass(createARMPromoteConstantPass());
if ((TM->getOptLevel() != CodeGenOptLevel::None &&
EnableGlobalMerge == cl::BOU_UNSET) ||
EnableGlobalMerge == cl::BOU_TRUE) {
diff --git a/llvm/lib/Target/ARM/CMakeLists.txt b/llvm/lib/Target/ARM/CMakeLists.txt
index d99368e1d3b2b..f3cec86ae5787 100644
--- a/llvm/lib/Target/ARM/CMakeLists.txt
+++ b/llvm/lib/Target/ARM/CMakeLists.txt
@@ -42,6 +42,7 @@ add_llvm_target(ARMCodeGen
ARMInstrInfo.cpp
ARMLegalizerInfo.cpp
ARMParallelDSP.cpp
+ ARMPromoteConstant.cpp
ARMLoadStoreOptimizer.cpp
ARMLowOverheadLoops.cpp
ARMBlockPlacement.cpp
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll b/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
index 7df230ccda67e..7b1ed93d05775 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-irtranslator.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple arm-unknown -mattr=+vfp2,+v4t -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LITTLE
-; RUN: llc -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -enable-arm-gisel-bigendian -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=BIG
+; RUN: llc -mtriple arm-unknown -mattr=+vfp2,+v4t,-neon,-mve,-mve.fp -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LITTLE
+; RUN: llc -mtriple armeb-unknown -mattr=+vfp2,+v4t,-neon,-mve,-mve.fp -global-isel -global-isel-abort=0 -enable-arm-gisel-bigendian -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=BIG
define void @test_void_return() {
; CHECK-LABEL: name: test_void_return
diff --git a/llvm/test/CodeGen/ARM/O3-pipeline.ll b/llvm/test/CodeGen/ARM/O3-pipeline.ll
index 9f4d70531a3f7..ff495f6a061f7 100644
--- a/llvm/test/CodeGen/ARM/O3-pipeline.ll
+++ b/llvm/test/CodeGen/ARM/O3-pipeline.ll
@@ -55,7 +55,12 @@
; CHECK-NEXT: CodeGen Prepare
; CHECK-NEXT: Dominator Tree Construction
; CHECK-NEXT: Exception handling preparation
+; CHECK-NEXT: ARM Promote Constant
+; CHECK-NEXT: FunctionPass Manager
+; CHECK-NEXT: Dominator Tree Construction
+; CHECK-NEXT: FunctionPass Manager
; CHECK-NEXT: Merge internal globals
+; CHECK-NEXT: Dominator Tree Construction
; CHECK-NEXT: Natural Loop Information
; CHECK-NEXT: Scalar Evolution Analysis
; CHECK-NEXT: Lazy Branch Probability Analysis
diff --git a/llvm/test/CodeGen/ARM/arm-promote-const.ll b/llvm/test/CodeGen/ARM/arm-promote-const.ll
new file mode 100644
index 0000000000000..c3344b2744c6e
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/arm-promote-const.ll
@@ -0,0 +1,181 @@
+; Disable machine cse to stress the different path of the algorithm.
+; Otherwise, we always fall in the simple case, i.e., only one definition.
+; RUN: llc < %s -mtriple=armv7-apple-ios7.0 -disable-machine-cse -arm-stress-promote-const -mattr=+neon -verify-machineinstrs | FileCheck -check-prefix=PROMOTED %s
+; The REGULAR run just checks that the inputs passed to promote const expose
+; the appropriate patterns.
+; RUN: llc < %s -mtriple=armv7-apple-ios7.0 -disable-machine-cse -arm-enable-promote-const=false -mattr=+neon -verify-machineinstrs | FileCheck -check-prefix=REGULAR %s
+
+%struct.uint8x16x4_t = type { [4 x <16 x i8>] }
+
+; Constant is a structure.
+define %struct.uint8x16x4_t @test1() {
+; PROMOTED-LABEL: _test1:
+; Promote constant has created a big constant for the whole structure.
+; PROMOTED: __PromotedConst
+; PROMOTED: vld1.8
+; PROMOTED: vld1.8
+; PROMOTED: vld1.8
+; PROMOTED: vld1.64
+; PROMOTED: bx lr
+
+; REGULAR-LABEL: _test1:
+; Regular access is quite bad, it performs 4 loads, one for each chunk of
+; the structure.
+; REGULAR: LCPI0_0
+; REGULAR: LCPI0_1
+; REGULAR: LCPI0_2
+; REGULAR: LCPI0_3
+; REGULAR: bx lr
+entry:
+ ret %struct.uint8x16x4_t { [4 x <16 x i8>] [<16 x i8> <i8 -40, i8 -93, i8 -118, i8 -99, i8 -75, i8 -105, i8 74, i8 -110, i8 62, i8 -115, i8 -119, i8 -120, i8 34, i8 -124, i8 0, i8 -128>, <16 x i8> <i8 32, i8 124, i8 121, i8 120, i8 8, i8 117, i8 -56, i8 113, i8 -76, i8 110, i8 -53, i8 107, i8 7, i8 105, i8 103, i8 102>, <16 x i8> <i8 -24, i8 99, i8 -121, i8 97, i8 66, i8 95, i8 24, i8 93, i8 6, i8 91, i8 12, i8 89, i8 39, i8 87, i8 86, i8 85>, <16 x i8> <i8 -104, i8 83, i8 -20, i8 81, i8 81, i8 80, i8 -59, i8 78, i8 73, i8 77, i8 -37, i8 75, i8 122, i8 74, i8 37, i8 73>] }
+}
+
+; Two different uses of the same constant in the same basic block.
+define <16 x i8> @test2(<16 x i8> %arg) {
+entry:
+; PROMOTED-LABEL: _test2:
+; In stress mode, constant vector are promoted.
+; PROMOTED: __PromotedConst.1
+; PROMOTED: vld1.64
+; PROMOTED: vadd.i8
+; PROMOTED: vmls.i8
+; PROMOTED: bx lr
+
+; REGULAR-LABEL: _test2:
+; Regular access is strictly the same as promoted access. The difference is
+; that the address, and thus the space in memory, is not shared between
+; constants.
+; REGULAR: LCPI1_0
+; REGULAR: vld1.64
+; REGULAR: vadd.i8
+; REGULAR: vmls.i8
+; REGULAR: bx lr
+ %add.i = add <16 x i8> %arg, <i8 -40, i8 -93, i8 -118, i8 -99, i8 -75, i8 -105, i8 74, i8 -110, i8 62, i8 -115, i8 -119, i8 -120, i8 34, i8 -124, i8 0, i8 -128>
+ %mul.i = mul <16 x i8> %add.i, <i8 -40, i8 -93, i8 -118, i8 -99, i8 -75, i8 -105, i8 74, i8 -110, i8 62, i8 -115, i8 -119, i8 -120, i8 34, i8 -124, i8 0, i8 -128>
+ %sub.i9 = sub <16 x i8> %add.i, %mul.i
+ ret <16 x i8> %sub.i9
+}
+
+; Two different uses of the same constant in two different basic blocks,
+; one dominates the other.
+define <16 x i8> @test3(<16 x i8> %arg, i32 %path) {
+; PROMOTED-LABEL: _test3:
+; In stress mode, constant vector are promoted.
+; Since the constant is the same as the previous function, the same address
+; must be used.
+; PROMOTED: __PromotedConst.1
+; PROMOTED: vld1.64
+; PROMOTED: __PromotedConst.1
+; PROMOTED: __PromotedConst.2
+; PROMOTED: vld1.64
+; PROMOTED-NOT: vld1.64
+; PROMOTED: bx lr
+
+; REGULAR-LABEL: _test3:
+; REGULAR: LCPI2_0
+; REGULAR: vld1.64
+; REGULAR: LCPI2_1
+; REGULAR: vld1.64
+; REGULAR-NOT: vld1.64
+; REGULAR: bx lr
+entry:
+ %add.i = add <16 x i8> %arg, <i8 -40, i8 -93, i8 -118, i8 -99, i8 -75, i8 -105, i8 74, i8 -110, i8 62, i8 -115, i8 -119, i8 -120, i8 34, i8 -124, i8 0, i8 -128>
+ %tobool = icmp eq i32 %path, 0
+ br i1 %tobool, label %if.else, label %if.then
+
+if.then:
+ %mul.i13 = mul <16 x i8> %add.i, <i8 -40, i8 -93, i8 -118, i8 -99, i8 -75, i8 -105, i8 74, i8 -110, i8 62, i8 -115, i8 -119, i8 -120, i8 34, i8 -124, i8 0, i8 -128>
+ br label %if.end
+
+if.else:
+ %mul.i = mul <16 x i8> %add.i, <i8 -24, i8 99, i8 -121, i8 97, i8 66, i8 95, i8 24, i8 93, i8 6, i8 91, i8 12, i8 89, i8 39, i8 87, i8 86, i8 85>
+ br label %if.end
+
+if.end:
+ %ret2.0 = phi <16 x i8> [ %mul.i13, %if.then ], [ %mul.i, %if.else ]
+ %add.i12 = add <16 x i8> %add.i, %ret2.0
+ ret <16 x i8> %add.i12
+}
+
+; Two different uses of the same constant in two different basic blocks,
+; none dominates the other.
+define <16 x i8> @test4(<16 x i8> %arg, i32 %path) {
+; PROMOTED-LABEL: _test4:
+; In stress mode, constant vector are promoted.
+; Since the constant is the same as the previous function, the same address
+; must be used.
+; PROMOTED: __PromotedConst.1
+; PROMOTED: vld1.64
+; PROMOTED-NOT: vld1.64
+; PROMOTED: bx lr
+
+; REGULAR-LABEL: _test4:
+; REGULAR: LCPI3_0
+; REGULAR: vld1.64
+; REGULAR-NOT: vld1.64
+; REGULAR: bx lr
+entry:
+ %add.i = add <16 x i8> %arg, <i8 -40, i8 -93, i8 -118, i8 -99, i8 -75, i8 -105, i8 74, i8 -110, i8 62, i8 -115, i8 -119, i8 -120, i8 34, i8 -124, i8 0, i8 -128>
+ %tobool = icmp eq i32 %path, 0
+ br i1 %tobool, label %if.end, label %if.then
+
+if.then:
+ %mul.i = mul <16 x i8> %add.i, <i8 -40, i8 -93, i8 -118, i8 -99, i8 -75, i8 -105, i8 74, i8 -110, i8 62, i8 -115, i8 -119, i8 -120, i8 34, i8 -124, i8 0, i8 -128>
+ br label %if.end
+
+if.end:
+ %ret.0 = phi <16 x i8> [ %mul.i, %if.then ], [ %add.i, %entry ]
+ ret <16 x i8> %ret.0
+}
+
+; Two different uses of the same constant in two different basic blocks,
+; one is in a phi.
+define <16 x i8> @test5(<16 x i8> %arg, i32 %path) {
+; PROMOTED-LABEL: _test5:
+; In stress mode, constant vector are promoted.
+; Since the constant is the same as the previous function, the same address
+; must be used.
+; PROMOTED: __PromotedConst.1
+; PROMOTED: vld1.64
+; PROMOTED-NOT: vld1.64
+; PROMOTED: bx lr
+
+; REGULAR-LABEL: _test5:
+; REGULAR: LCPI4_0
+; REGULAR: vld1.64
+; REGULAR: LCPI4_1
+; REGULAR: vld1.64
+; REGULAR: bx lr
+entry:
+ %tobool = icmp eq i32 %path, 0
+ br i1 %tobool, label %if.end, label %if.then
+
+if.then:
+ %add.i = add <16 x i8> %arg, <i8 -40, i8 -93, i8 -118, i8 -99, i8 -75, i8 -105, i8 74, i8 -110, i8 62, i8 -115, i8 -119, i8 -120, i8 34, i8 -124, i8 0, i8 -128>
+ %mul.i26 = mul <16 x i8> %add.i, <i8 -40, i8 -93, i8 -118, i8 -99, i8 -75, i8 -105, i8 74, i8 -110, i8 62, i8 -115, i8 -119, i8 -120, i8 34, i8 -124, i8 0, i8 -128>
+ br label %if.end
+
+if.end:
+ %ret.0 = phi <16 x i8> [ %mul.i26, %if.then ], [ <i8 -40, i8 -93, i8 -118, i8 -99, i8 -75, i8 -105, i8 74, i8 -110, i8 62, i8 -115, i8 -119, i8 -120, i8 34, i8 -124, i8 0, i8 -128>, %entry ]
+ %mul.i25 = mul <16 x i8> %ret.0, %ret.0
+ %mul.i24 = mul <16 x i8> %mul.i25, %mul.i25
+ %mul.i23 = mul <16 x i8> %mul.i24, %mul.i24
+ %mul.i = mul <16 x i8> %mul.i23, %mul.i23
+ ret <16 x i8> %mul.i
+}
+
+define void @accessBig(ptr %storage) {
+; PROMOTED-LABEL: _accessBig:
+; PROMOTED: __PromotedConst.3
+; PROMOTED: bx lr
+ store <1 x i80> <i80 483673642326615442599424>, ptr %storage
+ ret void
+}
+
+define void @asmStatement() {
+; PROMOTED-LABEL: _asmStatement:
+; PROMOTED-NOT: __PromotedConst
+; PROMOTED: bx lr
+ call void asm sideeffect "bfi r0, r0, $0, $1", "i,i"(i32 28, i32 4)
+ ret void
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn
index 42c6163263cff..286669205344f 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn
@@ -103,6 +103,7 @@ static_library("LLVMARMCodeGen") {
"ARMMacroFusion.cpp",
"ARMOptimizeBarriersPass.cpp",
"ARMParallelDSP.cpp",
+ "ARMPromoteConstant.cpp",
"ARMRegisterBankInfo.cpp",
"ARMRegisterInfo.cpp",
"ARMSLSHardening.cpp",
More information about the llvm-commits
mailing list