[llvm] [NewPM] Port x86-insert-vzero-upper (PR #181597)
Kyungtak Woo via llvm-commits
llvm-commits at lists.llvm.org
Sun Apr 19 19:51:01 PDT 2026
https://github.com/kevinwkt updated https://github.com/llvm/llvm-project/pull/181597
>From 32fc72011a9f887d0a3d0f7481152816e6625104 Mon Sep 17 00:00:00 2001
From: Kyungtak Woo <kevinwkt at google.com>
Date: Mon, 16 Feb 2026 04:46:04 +0000
Subject: [PATCH 1/8] rename issuevzeroupper to insertvzeroupper
---
llvm/lib/Target/X86/CMakeLists.txt | 2 +-
llvm/lib/Target/X86/X86.h | 7 +-
llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp | 3 +-
...VZeroUpper.cpp => X86InsertVZeroUpper.cpp} | 183 ++++++++++--------
llvm/lib/Target/X86/X86PassRegistry.def | 2 +-
llvm/lib/Target/X86/X86TargetMachine.cpp | 2 +-
llvm/test/CodeGen/X86/llc-pipeline-npm.ll | 4 +
7 files changed, 111 insertions(+), 92 deletions(-)
rename llvm/lib/Target/X86/{X86VZeroUpper.cpp => X86InsertVZeroUpper.cpp} (66%)
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index c2dd34efb844d..c031d4bfa7389 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -87,7 +87,7 @@ set(sources
X86TargetMachine.cpp
X86TargetObjectFile.cpp
X86TargetTransformInfo.cpp
- X86VZeroUpper.cpp
+ X86InsertVZeroUpper.cpp
X86WinEHState.cpp
X86WinEHUnwindV2.cpp
GISel/X86CallLowering.cpp
diff --git a/llvm/lib/Target/X86/X86.h b/llvm/lib/Target/X86/X86.h
index 5ff33a48b63ae..3b31dee767fae 100644
--- a/llvm/lib/Target/X86/X86.h
+++ b/llvm/lib/Target/X86/X86.h
@@ -69,8 +69,13 @@ FunctionPass *createX86FPStackifierLegacyPass();
/// This pass inserts AVX vzeroupper instructions before each call to avoid
/// transition penalty between functions encoded with AVX and SSE.
-FunctionPass *createX86IssueVZeroUpperPass();
+class X86InsertVZeroUpperPass : public PassInfoMixin<X86InsertVZeroUpperPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+FunctionPass *createX86InsertVZeroUpperLegacyPass();
/// This pass inserts ENDBR instructions before indirect jump/call
/// destinations as part of CET IBT mechanism.
class X86IndirectBranchTrackingPass
diff --git a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
index 7bc009f24a412..524cde4a8d4df 100644
--- a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
+++ b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
@@ -179,8 +179,7 @@ void X86CodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) const {
}
addMachineFunctionPass(X86IndirectBranchTrackingPass(), PMW);
- // TODO(boomanaiden154): Add X86IssueVZeroUpperPass here once it has been
- // ported.
+ addMachineFunctionPass(X86InsertVZeroUpperPass(), PMW);
if (getOptLevel() != CodeGenOptLevel::None) {
addMachineFunctionPass(X86FixupBWInstsPass(), PMW);
diff --git a/llvm/lib/Target/X86/X86VZeroUpper.cpp b/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
similarity index 66%
rename from llvm/lib/Target/X86/X86VZeroUpper.cpp
rename to llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
index 2f28ab36aa193..703a4a5da0471 100644
--- a/llvm/lib/Target/X86/X86VZeroUpper.cpp
+++ b/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
@@ -1,4 +1,4 @@
-//===- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter ------------===//
+//===---- X86InsertVZeroUpper.cpp - AVX vzeroupper instruction inserter ---===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -24,9 +24,11 @@
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/IR/Analysis.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/Function.h"
@@ -37,91 +39,78 @@
using namespace llvm;
-#define DEBUG_TYPE "x86-vzeroupper"
+#define DEBUG_TYPE "x86-insert-vzero-upper"
static cl::opt<bool>
-UseVZeroUpper("x86-use-vzeroupper", cl::Hidden,
- cl::desc("Minimize AVX to SSE transition penalty"),
- cl::init(true));
+ UseVZeroUpper("x86-use-vzeroupper", cl::Hidden,
+ cl::desc("Minimize AVX to SSE transition penalty"),
+ cl::init(true));
STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
namespace {
+class X86InsertVZeroUpperLegacy : public MachineFunctionPass {
+public:
+ static char ID;
- class VZeroUpperInserter : public MachineFunctionPass {
- public:
- VZeroUpperInserter() : MachineFunctionPass(ID) {}
+ X86InsertVZeroUpperLegacy() : MachineFunctionPass(ID) {}
- bool runOnMachineFunction(MachineFunction &MF) override;
+ StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
- MachineFunctionProperties getRequiredProperties() const override {
- return MachineFunctionProperties().setNoVRegs();
- }
-
- StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
-
- private:
- void processBasicBlock(MachineBasicBlock &MBB);
- void insertVZeroUpper(MachineBasicBlock::iterator I,
- MachineBasicBlock &MBB);
- void addDirtySuccessor(MachineBasicBlock &MBB);
-
- enum BlockExitState { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
-
- static const char* getBlockExitStateName(BlockExitState ST);
-
- // Core algorithm state:
- // BlockState - Each block is either:
- // - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
- // vzeroupper instructions in this block.
- // - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
- // block that will ensure that YMM/ZMM is clean on exit.
- // - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
- // subsequent vzeroupper in the block clears it.
- //
- // AddedToDirtySuccessors - This flag is raised when a block is added to the
- // DirtySuccessors list to ensure that it's not
- // added multiple times.
- //
- // FirstUnguardedCall - Records the location of the first unguarded call in
- // each basic block that may need to be guarded by a
- // vzeroupper. We won't know whether it actually needs
- // to be guarded until we discover a predecessor that
- // is DIRTY_OUT.
- struct BlockState {
- BlockExitState ExitState = PASS_THROUGH;
- bool AddedToDirtySuccessors = false;
- MachineBasicBlock::iterator FirstUnguardedCall;
-
- BlockState() = default;
- };
-
- using BlockStateMap = SmallVector<BlockState, 8>;
- using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
-
- BlockStateMap BlockStates;
- DirtySuccessorsWorkList DirtySuccessors;
- bool EverMadeChange;
- bool IsX86INTR;
- const TargetInstrInfo *TII;
-
- static char ID;
- };
+ bool runOnMachineFunction(MachineFunction &MF) override;
+ MachineFunctionProperties getRequiredProperties() const override {
+ return MachineFunctionProperties().setNoVRegs();
+ }
+};
+
+enum BlockExitState { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
+
+// Core algorithm state:
+// BlockState - Each block is either:
+// - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
+// vzeroupper instructions in this block.
+// - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
+// block that will ensure that YMM/ZMM is clean on exit.
+// - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
+// subsequent vzeroupper in the block clears it.
+//
+// AddedToDirtySuccessors - This flag is raised when a block is added to the
+// DirtySuccessors list to ensure that it's not
+// added multiple times.
+//
+// FirstUnguardedCall - Records the location of the first unguarded call in
+// each basic block that may need to be guarded by a
+// vzeroupper. We won't know whether it actually needs
+// to be guarded until we discover a predecessor that
+// is DIRTY_OUT.
+struct BlockState {
+ BlockExitState ExitState = PASS_THROUGH;
+ bool AddedToDirtySuccessors = false;
+ MachineBasicBlock::iterator FirstUnguardedCall;
+
+ BlockState() = default;
+};
+
+using BlockStateMap = SmallVector<BlockState, 8>;
+using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
} // end anonymous namespace
-char VZeroUpperInserter::ID = 0;
+char X86InsertVZeroUpperLegacy::ID = 0;
-FunctionPass *llvm::createX86IssueVZeroUpperPass() {
- return new VZeroUpperInserter();
+FunctionPass *llvm::createX86InsertVZeroUpperLegacyPass() {
+ return new X86InsertVZeroUpperLegacy();
}
#ifndef NDEBUG
-const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
+static const char *getBlockExitStateName(BlockExitState ST) {
switch (ST) {
- case PASS_THROUGH: return "Pass-through";
- case EXITS_DIRTY: return "Exits-dirty";
- case EXITS_CLEAN: return "Exits-clean";
+ case PASS_THROUGH:
+ return "Pass-through";
+ case EXITS_DIRTY:
+ return "Exits-dirty";
+ case EXITS_CLEAN:
+ return "Exits-clean";
}
llvm_unreachable("Invalid block exit state.");
}
@@ -179,15 +168,18 @@ static bool callHasRegMask(MachineInstr &MI) {
}
/// Insert a vzeroupper instruction before I.
-void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
- MachineBasicBlock &MBB) {
+static void insertVZeroUpper(MachineBasicBlock::iterator I,
+ MachineBasicBlock &MBB, const TargetInstrInfo *TII,
+ bool &EverMadeChange) {
BuildMI(MBB, I, I->getDebugLoc(), TII->get(X86::VZEROUPPER));
++NumVZU;
EverMadeChange = true;
}
/// Add MBB to the DirtySuccessors list if it hasn't already been added.
-void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
+static void addDirtySuccessor(MachineBasicBlock &MBB,
+ BlockStateMap &BlockStates,
+ DirtySuccessorsWorkList &DirtySuccessors) {
if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
DirtySuccessors.push_back(&MBB);
BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
@@ -196,7 +188,11 @@ void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
/// Loop over all of the instructions in the basic block, inserting vzeroupper
/// instructions before function calls.
-void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
+static void processBasicBlock(MachineBasicBlock &MBB,
+ BlockStateMap &BlockStates,
+ DirtySuccessorsWorkList &DirtySuccessors,
+ bool IsX86INTR, const TargetInstrInfo *TII,
+ bool &EverMadeChange) {
// Start by assuming that the block is PASS_THROUGH which implies no unguarded
// calls.
BlockExitState CurState = PASS_THROUGH;
@@ -253,7 +249,7 @@ void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
// After the inserted VZEROUPPER the state becomes clean again, but
// other YMM/ZMM may appear before other subsequent calls or even before
// the end of the BB.
- insertVZeroUpper(MI, MBB);
+ insertVZeroUpper(MI, MBB, TII, EverMadeChange);
CurState = EXITS_CLEAN;
} else if (CurState == PASS_THROUGH) {
// If this block is currently in pass-through state and we encounter a
@@ -271,24 +267,22 @@ void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
if (CurState == EXITS_DIRTY)
for (MachineBasicBlock *Succ : MBB.successors())
- addDirtySuccessor(*Succ);
+ addDirtySuccessor(*Succ, BlockStates, DirtySuccessors);
BlockStates[MBB.getNumber()].ExitState = CurState;
}
/// Loop over all of the basic blocks, inserting vzeroupper instructions before
/// function calls.
-bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
+static bool insertVZeroUpper(MachineFunction &MF) {
if (!UseVZeroUpper)
return false;
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
if (!ST.hasAVX() || !ST.insertVZEROUPPER())
return false;
- TII = ST.getInstrInfo();
+
MachineRegisterInfo &MRI = MF.getRegInfo();
- EverMadeChange = false;
- IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);
@@ -309,20 +303,26 @@ bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
if (!YmmOrZmmUsed)
return false;
- assert(BlockStates.empty() && DirtySuccessors.empty() &&
+ const TargetInstrInfo *TII = ST.getInstrInfo();
+ bool IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
+ bool EverMadeChange = false;
+ BlockStateMap BlockStates(MF.getNumBlockIDs());
+ DirtySuccessorsWorkList DirtySuccessors;
+
+ assert(BlockStates.size() == MF.getNumBlockIDs() && DirtySuccessors.empty() &&
"X86VZeroUpper state should be clear");
- BlockStates.resize(MF.getNumBlockIDs());
// Process all blocks. This will compute block exit states, record the first
// unguarded call in each block, and add successors of dirty blocks to the
// DirtySuccessors list.
for (MachineBasicBlock &MBB : MF)
- processBasicBlock(MBB);
+ processBasicBlock(MBB, BlockStates, DirtySuccessors, IsX86INTR, TII,
+ EverMadeChange);
// If any YMM/ZMM regs are live-in to this function, add the entry block to
// the DirtySuccessors list
if (FnHasLiveInYmmOrZmm)
- addDirtySuccessor(MF.front());
+ addDirtySuccessor(MF.front(), BlockStates, DirtySuccessors);
// Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
// vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
@@ -335,7 +335,7 @@ bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
// MBB is a successor of a dirty block, so its first call needs to be
// guarded.
if (BBState.FirstUnguardedCall != MBB.end())
- insertVZeroUpper(BBState.FirstUnguardedCall, MBB);
+ insertVZeroUpper(BBState.FirstUnguardedCall, MBB, TII, EverMadeChange);
// If this successor was a pass-through block, then it is now dirty. Its
// successors need to be added to the worklist (if they haven't been
@@ -344,10 +344,21 @@ bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
<< " was Pass-through, is now Dirty-out.\n");
for (MachineBasicBlock *Succ : MBB.successors())
- addDirtySuccessor(*Succ);
+ addDirtySuccessor(*Succ, BlockStates, DirtySuccessors);
}
}
- BlockStates.clear();
return EverMadeChange;
}
+
+bool X86InsertVZeroUpperLegacy::runOnMachineFunction(MachineFunction &MF) {
+ return insertVZeroUpper(MF);
+}
+
+PreservedAnalyses
+X86InsertVZeroUpperPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ return insertVZeroUpper(MF) ? getMachineFunctionPassPreservedAnalyses()
+ .preserveSet<CFGAnalyses>()
+ : PreservedAnalyses::all();
+}
\ No newline at end of file
diff --git a/llvm/lib/Target/X86/X86PassRegistry.def b/llvm/lib/Target/X86/X86PassRegistry.def
index be85b7cf216c8..5d0d15ab1b90d 100644
--- a/llvm/lib/Target/X86/X86PassRegistry.def
+++ b/llvm/lib/Target/X86/X86PassRegistry.def
@@ -50,6 +50,7 @@ MACHINE_FUNCTION_PASS("x86-flags-copy-lowering", X86FlagsCopyLoweringPass())
MACHINE_FUNCTION_PASS("x86-fp-stackifier", X86FPStackifierPass())
MACHINE_FUNCTION_PASS("x86-global-base-reg", X86GlobalBaseRegPass())
MACHINE_FUNCTION_PASS("x86-indirect-branch-tracking", X86IndirectBranchTrackingPass())
+MACHINE_FUNCTION_PASS("x86-insert-vzero-upper", X86InsertVZeroUpperPass())
MACHINE_FUNCTION_PASS("x86-insert-x87-wait", X86InsertX87WaitPass())
MACHINE_FUNCTION_PASS("x86-isel", X86ISelDAGToDAGPass(*this))
MACHINE_FUNCTION_PASS("x86-lower-tile-copy", X86LowerTileCopyPass())
@@ -71,6 +72,5 @@ MACHINE_FUNCTION_PASS("x86-wineh-unwindv2", X86WinEHUnwindV2Pass())
#endif
DUMMY_MACHINE_FUNCTION_PASS("x86-execution-domain-fix", X86ExecutionDomainFix())
DUMMY_MACHINE_FUNCTION_PASS("x86-indirect-thunks", X86IndirectThunks())
-DUMMY_MACHINE_FUNCTION_PASS("x86-issue-vzero-upper", X86IssueVZeroUpperPass())
DUMMY_MACHINE_FUNCTION_PASS("x86-pad-short-functions", X86PadShortFunctionsPass())
#undef DUMMY_MACHINE_FUNCTION_PASS
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index eba9486745f54..4ddadbec5a174 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -565,7 +565,7 @@ void X86PassConfig::addPreEmitPass() {
addPass(createX86IndirectBranchTrackingLegacyPass());
- addPass(createX86IssueVZeroUpperPass());
+ addPass(createX86InsertVZeroUpperLegacyPass());
if (getOptLevel() != CodeGenOptLevel::None) {
addPass(createX86FixupBWInstsLegacyPass());
diff --git a/llvm/test/CodeGen/X86/llc-pipeline-npm.ll b/llvm/test/CodeGen/X86/llc-pipeline-npm.ll
index e392cf7408301..0893af972c7f9 100644
--- a/llvm/test/CodeGen/X86/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/X86/llc-pipeline-npm.ll
@@ -58,6 +58,7 @@
; O0-NEXT: xray-instrumentation
; O0-NEXT: patchable-function
; O0-NEXT: x86-indirect-branch-tracking
+; O0-NEXT: x86-insert-vzero-upper
; O0-NEXT: x86-compress-evex
; O0-NEXT: x86-insert-x87-wait
; O0-NEXT: FuncletLayoutPass
@@ -172,6 +173,7 @@
; O2-NEXT: patchable-function
; O2-NEXT: BreakFalseDepsPass
; O2-NEXT: x86-indirect-branch-tracking
+; O2-NEXT: x86-insert-vzero-upper
; O2-NEXT: x86-fixup-bw-insts
; O2-NEXT: x86-fixup-leas
; O2-NEXT: x86-fixup-inst-tuning
@@ -242,6 +244,7 @@
; O0-WINDOWS-NEXT: xray-instrumentation
; O0-WINDOWS-NEXT: patchable-function
; O0-WINDOWS-NEXT: x86-indirect-branch-tracking
+; O0-WINDOWS-NEXT: x86-insert-vzero-upper
; O0-WINDOWS-NEXT: x86-compress-evex
; O0-WINDOWS-NEXT: x86-insert-x87-wait
; O0-WINDOWS-NEXT: FuncletLayoutPass
@@ -359,6 +362,7 @@
; O3-WINDOWS-NEXT: patchable-function
; O3-WINDOWS-NEXT: BreakFalseDepsPass
; O3-WINDOWS-NEXT: x86-indirect-branch-tracking
+; O3-WINDOWS-NEXT: x86-insert-vzero-upper
; O3-WINDOWS-NEXT: x86-fixup-bw-insts
; O3-WINDOWS-NEXT: x86-fixup-leas
; O3-WINDOWS-NEXT: x86-fixup-inst-tuning
>From 97f9af9d8ff09bf74fe147121d948447aac79ba8 Mon Sep 17 00:00:00 2001
From: Kyungtak Woo <kevinwkt at google.com>
Date: Mon, 16 Feb 2026 17:59:58 +0000
Subject: [PATCH 2/8] alpha sort the list
---
llvm/lib/Target/X86/CMakeLists.txt | 44 +++++++++++++++---------------
1 file changed, 22 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index c031d4bfa7389..b795077602fc8 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -27,75 +27,75 @@ tablegen(LLVM X86GenPostLegalizeGICombiner.inc -gen-global-isel-combiner
add_public_tablegen_target(X86CommonTableGen)
set(sources
+ GISel/X86CallLowering.cpp
+ GISel/X86InstructionSelector.cpp
+ GISel/X86LegalizerInfo.cpp
+ GISel/X86PostLegalizerCombiner.cpp
+ GISel/X86PreLegalizerCombiner.cpp
+ GISel/X86RegisterBankInfo.cpp
X86ArgumentStackSlotRebase.cpp
X86AsmPrinter.cpp
+ X86AvoidStoreForwardingBlocks.cpp
X86AvoidTrailingCall.cpp
X86CallFrameOptimization.cpp
X86CallingConv.cpp
X86CleanupLocalDynamicTLS.cpp
X86CmovConversion.cpp
X86CodeGenPassBuilder.cpp
+ X86CompressEVEX.cpp
X86DomainReassignment.cpp
- X86GlobalBaseReg.cpp
- X86LowerTileCopy.cpp
- X86LowerAMXType.cpp
- X86LowerAMXIntrinsics.cpp
- X86TileConfig.cpp
- X86FastPreTileConfig.cpp
- X86FastTileConfig.cpp
- X86PreTileConfig.cpp
+ X86DynAllocaExpander.cpp
X86ExpandPseudo.cpp
X86FastISel.cpp
+ X86FastPreTileConfig.cpp
+ X86FastTileConfig.cpp
X86FixupBWInsts.cpp
- X86FixupLEAs.cpp
X86FixupInstTuning.cpp
- X86FixupVectorConstants.cpp
- X86AvoidStoreForwardingBlocks.cpp
- X86DynAllocaExpander.cpp
+ X86FixupLEAs.cpp
X86FixupSetCC.cpp
+ X86FixupVectorConstants.cpp
X86FlagsCopyLowering.cpp
X86FloatingPoint.cpp
X86FrameLowering.cpp
+ X86GlobalBaseReg.cpp
X86ISelDAGToDAG.cpp
X86ISelLowering.cpp
X86ISelLoweringCall.cpp
X86IndirectBranchTracking.cpp
X86IndirectThunks.cpp
+ X86InsertVZeroUpper.cpp
X86InsertX87Wait.cpp
- X86InterleavedAccess.cpp
X86InstCombineIntrinsic.cpp
X86InstrFMA3Info.cpp
X86InstrFoldTables.cpp
X86InstrInfo.cpp
- X86CompressEVEX.cpp
+ X86InterleavedAccess.cpp
X86LoadValueInjectionLoadHardening.cpp
X86LoadValueInjectionRetHardening.cpp
+ X86LowerAMXIntrinsics.cpp
+ X86LowerAMXType.cpp
+ X86LowerTileCopy.cpp
X86MCInstLower.cpp
X86MachineFunctionInfo.cpp
X86MacroFusion.cpp
X86OptimizeLEAs.cpp
X86PadShortFunction.cpp
X86PartialReduction.cpp
+ X86PreTileConfig.cpp
X86RegisterInfo.cpp
X86ReturnThunks.cpp
X86SelectionDAGInfo.cpp
X86ShuffleDecodeConstantPool.cpp
- X86SpeculativeLoadHardening.cpp
X86SpeculativeExecutionSideEffectSuppression.cpp
+ X86SpeculativeLoadHardening.cpp
X86Subtarget.cpp
X86SuppressAPXForReloc.cpp
X86TargetMachine.cpp
X86TargetObjectFile.cpp
X86TargetTransformInfo.cpp
- X86InsertVZeroUpper.cpp
+ X86TileConfig.cpp
X86WinEHState.cpp
X86WinEHUnwindV2.cpp
- GISel/X86CallLowering.cpp
- GISel/X86InstructionSelector.cpp
- GISel/X86LegalizerInfo.cpp
- GISel/X86PostLegalizerCombiner.cpp
- GISel/X86PreLegalizerCombiner.cpp
- GISel/X86RegisterBankInfo.cpp
)
add_llvm_target(X86CodeGen ${sources}
>From e692f6a5314ecd49f09f3f86f1efd3635ac31fe9 Mon Sep 17 00:00:00 2001
From: Kyungtak Woo <kevinwkt at google.com>
Date: Mon, 16 Feb 2026 19:41:34 +0000
Subject: [PATCH 3/8] fix eof and some sorting
---
llvm/lib/Target/X86/CMakeLists.txt | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index b795077602fc8..916571a40a7cc 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -27,12 +27,6 @@ tablegen(LLVM X86GenPostLegalizeGICombiner.inc -gen-global-isel-combiner
add_public_tablegen_target(X86CommonTableGen)
set(sources
- GISel/X86CallLowering.cpp
- GISel/X86InstructionSelector.cpp
- GISel/X86LegalizerInfo.cpp
- GISel/X86PostLegalizerCombiner.cpp
- GISel/X86PreLegalizerCombiner.cpp
- GISel/X86RegisterBankInfo.cpp
X86ArgumentStackSlotRebase.cpp
X86AsmPrinter.cpp
X86AvoidStoreForwardingBlocks.cpp
@@ -96,6 +90,12 @@ set(sources
X86TileConfig.cpp
X86WinEHState.cpp
X86WinEHUnwindV2.cpp
+ GISel/X86CallLowering.cpp
+ GISel/X86InstructionSelector.cpp
+ GISel/X86LegalizerInfo.cpp
+ GISel/X86PostLegalizerCombiner.cpp
+ GISel/X86PreLegalizerCombiner.cpp
+ GISel/X86RegisterBankInfo.cpp
)
add_llvm_target(X86CodeGen ${sources}
@@ -129,4 +129,4 @@ add_subdirectory(AsmParser)
add_subdirectory(Disassembler)
add_subdirectory(MCA)
add_subdirectory(MCTargetDesc)
-add_subdirectory(TargetInfo)
+add_subdirectory(TargetInfo)
\ No newline at end of file
>From d447507b2bd7307c6cfb09f81fac9e80a4dacf32 Mon Sep 17 00:00:00 2001
From: Kyungtak Woo <kevinwkt at google.com>
Date: Mon, 16 Feb 2026 19:44:30 +0000
Subject: [PATCH 4/8] fix more eof
---
llvm/lib/Target/X86/CMakeLists.txt | 2 +-
llvm/lib/Target/X86/X86InsertVZeroUpper.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index 916571a40a7cc..a62ed877cb04a 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -129,4 +129,4 @@ add_subdirectory(AsmParser)
add_subdirectory(Disassembler)
add_subdirectory(MCA)
add_subdirectory(MCTargetDesc)
-add_subdirectory(TargetInfo)
\ No newline at end of file
+add_subdirectory(TargetInfo)
diff --git a/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp b/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
index 703a4a5da0471..7596bc4948f01 100644
--- a/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
+++ b/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
@@ -361,4 +361,4 @@ X86InsertVZeroUpperPass::run(MachineFunction &MF,
return insertVZeroUpper(MF) ? getMachineFunctionPassPreservedAnalyses()
.preserveSet<CFGAnalyses>()
: PreservedAnalyses::all();
-}
\ No newline at end of file
+}
>From be7e8ba650d5925cb8799834a8cbd01bad2656cf Mon Sep 17 00:00:00 2001
From: Kyungtak Woo <kevinwkt at google.com>
Date: Mon, 16 Feb 2026 19:57:19 +0000
Subject: [PATCH 5/8] alpha only the new file
---
llvm/lib/Target/X86/CMakeLists.txt | 32 +++++++++++++++---------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index a62ed877cb04a..45d77323a1005 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -29,29 +29,33 @@ add_public_tablegen_target(X86CommonTableGen)
set(sources
X86ArgumentStackSlotRebase.cpp
X86AsmPrinter.cpp
- X86AvoidStoreForwardingBlocks.cpp
X86AvoidTrailingCall.cpp
X86CallFrameOptimization.cpp
X86CallingConv.cpp
X86CleanupLocalDynamicTLS.cpp
X86CmovConversion.cpp
X86CodeGenPassBuilder.cpp
- X86CompressEVEX.cpp
X86DomainReassignment.cpp
- X86DynAllocaExpander.cpp
- X86ExpandPseudo.cpp
- X86FastISel.cpp
+ X86GlobalBaseReg.cpp
+ X86LowerTileCopy.cpp
+ X86LowerAMXType.cpp
+ X86LowerAMXIntrinsics.cpp
+ X86TileConfig.cpp
X86FastPreTileConfig.cpp
X86FastTileConfig.cpp
+ X86PreTileConfig.cpp
+ X86ExpandPseudo.cpp
+ X86FastISel.cpp
X86FixupBWInsts.cpp
- X86FixupInstTuning.cpp
X86FixupLEAs.cpp
- X86FixupSetCC.cpp
+ X86FixupInstTuning.cpp
X86FixupVectorConstants.cpp
+ X86AvoidStoreForwardingBlocks.cpp
+ X86DynAllocaExpander.cpp
+ X86FixupSetCC.cpp
X86FlagsCopyLowering.cpp
X86FloatingPoint.cpp
X86FrameLowering.cpp
- X86GlobalBaseReg.cpp
X86ISelDAGToDAG.cpp
X86ISelLowering.cpp
X86ISelLoweringCall.cpp
@@ -59,35 +63,31 @@ set(sources
X86IndirectThunks.cpp
X86InsertVZeroUpper.cpp
X86InsertX87Wait.cpp
+ X86InterleavedAccess.cpp
X86InstCombineIntrinsic.cpp
X86InstrFMA3Info.cpp
X86InstrFoldTables.cpp
X86InstrInfo.cpp
- X86InterleavedAccess.cpp
+ X86CompressEVEX.cpp
X86LoadValueInjectionLoadHardening.cpp
X86LoadValueInjectionRetHardening.cpp
- X86LowerAMXIntrinsics.cpp
- X86LowerAMXType.cpp
- X86LowerTileCopy.cpp
X86MCInstLower.cpp
X86MachineFunctionInfo.cpp
X86MacroFusion.cpp
X86OptimizeLEAs.cpp
X86PadShortFunction.cpp
X86PartialReduction.cpp
- X86PreTileConfig.cpp
X86RegisterInfo.cpp
X86ReturnThunks.cpp
X86SelectionDAGInfo.cpp
X86ShuffleDecodeConstantPool.cpp
- X86SpeculativeExecutionSideEffectSuppression.cpp
X86SpeculativeLoadHardening.cpp
+ X86SpeculativeExecutionSideEffectSuppression.cpp
X86Subtarget.cpp
X86SuppressAPXForReloc.cpp
X86TargetMachine.cpp
X86TargetObjectFile.cpp
X86TargetTransformInfo.cpp
- X86TileConfig.cpp
X86WinEHState.cpp
X86WinEHUnwindV2.cpp
GISel/X86CallLowering.cpp
@@ -129,4 +129,4 @@ add_subdirectory(AsmParser)
add_subdirectory(Disassembler)
add_subdirectory(MCA)
add_subdirectory(MCTargetDesc)
-add_subdirectory(TargetInfo)
+add_subdirectory(TargetInfo)
\ No newline at end of file
>From 5cd0f5cc08eeb741cca540fa60114815814d53fc Mon Sep 17 00:00:00 2001
From: Kyungtak Woo <kevinwkt at google.com>
Date: Mon, 16 Feb 2026 19:58:04 +0000
Subject: [PATCH 6/8] eof
---
llvm/lib/Target/X86/CMakeLists.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt
index 45d77323a1005..e81d0b29f019e 100644
--- a/llvm/lib/Target/X86/CMakeLists.txt
+++ b/llvm/lib/Target/X86/CMakeLists.txt
@@ -129,4 +129,4 @@ add_subdirectory(AsmParser)
add_subdirectory(Disassembler)
add_subdirectory(MCA)
add_subdirectory(MCTargetDesc)
-add_subdirectory(TargetInfo)
\ No newline at end of file
+add_subdirectory(TargetInfo)
>From e189f048b5e297a7714ce3e78ba9e935efce140e Mon Sep 17 00:00:00 2001
From: Kyungtak Woo <kevinwkt at google.com>
Date: Mon, 6 Apr 2026 04:46:38 +0000
Subject: [PATCH 7/8] Correct DEBUG_TYPE string in X86InsertVZeroUpper to match
pass name
---
llvm/lib/Target/X86/X86InsertVZeroUpper.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp b/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
index 7596bc4948f01..954e0e2baa24e 100644
--- a/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
+++ b/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
@@ -39,7 +39,7 @@
using namespace llvm;
-#define DEBUG_TYPE "x86-insert-vzero-upper"
+#define DEBUG_TYPE "x86-insert-vzeroupper"
static cl::opt<bool>
UseVZeroUpper("x86-use-vzeroupper", cl::Hidden,
>From d8d588f896e01805355526e297fe4eb94eae80f6 Mon Sep 17 00:00:00 2001
From: Kyungtak Woo <kevinwkt at google.com>
Date: Wed, 8 Apr 2026 02:20:36 +0000
Subject: [PATCH 8/8] update header desc and insertVZeroUpper returns bool
---
llvm/lib/Target/X86/X86InsertVZeroUpper.cpp | 38 +++++++++++----------
1 file changed, 20 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp b/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
index 954e0e2baa24e..4cbb911a5880f 100644
--- a/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
+++ b/llvm/lib/Target/X86/X86InsertVZeroUpper.cpp
@@ -1,16 +1,17 @@
-//===---- X86InsertVZeroUpper.cpp - AVX vzeroupper instruction inserter ---===//
+//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
-//
-// This file defines the pass which inserts x86 AVX vzeroupper instructions
-// before calls to SSE encoded functions. This avoids transition latency
-// penalty when transferring control between AVX encoded instructions and old
-// SSE encoding mode.
-//
+///
+/// \file
+/// This file defines the pass which inserts x86 AVX vzeroupper instructions
+/// before calls to SSE encoded functions. This avoids transition latency
+/// penalty when transferring control between AVX encoded instructions and old
+/// SSE encoding mode.
+///
//===----------------------------------------------------------------------===//
#include "X86.h"
@@ -168,12 +169,12 @@ static bool callHasRegMask(MachineInstr &MI) {
}
/// Insert a vzeroupper instruction before I.
-static void insertVZeroUpper(MachineBasicBlock::iterator I,
- MachineBasicBlock &MBB, const TargetInstrInfo *TII,
- bool &EverMadeChange) {
+static bool insertVZeroUpper(MachineBasicBlock::iterator I,
+ MachineBasicBlock &MBB,
+ const TargetInstrInfo *TII) {
BuildMI(MBB, I, I->getDebugLoc(), TII->get(X86::VZEROUPPER));
++NumVZU;
- EverMadeChange = true;
+ return true;
}
/// Add MBB to the DirtySuccessors list if it hasn't already been added.
@@ -188,15 +189,15 @@ static void addDirtySuccessor(MachineBasicBlock &MBB,
/// Loop over all of the instructions in the basic block, inserting vzeroupper
/// instructions before function calls.
-static void processBasicBlock(MachineBasicBlock &MBB,
+static bool processBasicBlock(MachineBasicBlock &MBB,
BlockStateMap &BlockStates,
DirtySuccessorsWorkList &DirtySuccessors,
- bool IsX86INTR, const TargetInstrInfo *TII,
- bool &EverMadeChange) {
+ bool IsX86INTR, const TargetInstrInfo *TII) {
// Start by assuming that the block is PASS_THROUGH which implies no unguarded
// calls.
BlockExitState CurState = PASS_THROUGH;
BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
+ bool MadeChange = false;
for (MachineInstr &MI : MBB) {
bool IsCall = MI.isCall();
@@ -249,7 +250,7 @@ static void processBasicBlock(MachineBasicBlock &MBB,
// After the inserted VZEROUPPER the state becomes clean again, but
// other YMM/ZMM may appear before other subsequent calls or even before
// the end of the BB.
- insertVZeroUpper(MI, MBB, TII, EverMadeChange);
+ MadeChange |= insertVZeroUpper(MI, MBB, TII);
CurState = EXITS_CLEAN;
} else if (CurState == PASS_THROUGH) {
// If this block is currently in pass-through state and we encounter a
@@ -270,6 +271,7 @@ static void processBasicBlock(MachineBasicBlock &MBB,
addDirtySuccessor(*Succ, BlockStates, DirtySuccessors);
BlockStates[MBB.getNumber()].ExitState = CurState;
+ return MadeChange;
}
/// Loop over all of the basic blocks, inserting vzeroupper instructions before
@@ -316,8 +318,8 @@ static bool insertVZeroUpper(MachineFunction &MF) {
// unguarded call in each block, and add successors of dirty blocks to the
// DirtySuccessors list.
for (MachineBasicBlock &MBB : MF)
- processBasicBlock(MBB, BlockStates, DirtySuccessors, IsX86INTR, TII,
- EverMadeChange);
+ EverMadeChange |=
+ processBasicBlock(MBB, BlockStates, DirtySuccessors, IsX86INTR, TII);
// If any YMM/ZMM regs are live-in to this function, add the entry block to
// the DirtySuccessors list
@@ -335,7 +337,7 @@ static bool insertVZeroUpper(MachineFunction &MF) {
// MBB is a successor of a dirty block, so its first call needs to be
// guarded.
if (BBState.FirstUnguardedCall != MBB.end())
- insertVZeroUpper(BBState.FirstUnguardedCall, MBB, TII, EverMadeChange);
+ EverMadeChange |= insertVZeroUpper(BBState.FirstUnguardedCall, MBB, TII);
// If this successor was a pass-through block, then it is now dirty. Its
// successors need to be added to the worklist (if they haven't been
More information about the llvm-commits
mailing list