[llvm] [X86][APX] Reuse EFLAGS across multi-predecessor blocks via NF in optimizeCompareInstr (PR #208184)
Feng Zou via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 00:39:03 PDT 2026
https://github.com/fzou1 updated https://github.com/llvm/llvm-project/pull/208184
>From 87c8ef271c7c9780eaf967c6e5ce7387464d1ff1 Mon Sep 17 00:00:00 2001
From: Feng Zou <feng.zou at intel.com>
Date: Wed, 8 Jul 2026 12:36:25 +0800
Subject: [PATCH 1/4] [X86] Reuse EFLAGS across multi-predecessor blocks via NF
in optimizeCompareInstr
Extend optimizeCompareInstr to reuse EFLAGS produced in predecessor
blocks (including via NF variants) when a compare is redundant, covering
multi-predecessor, dominating, and cyclic CFGs. Share the NF-clobber
removal helper with FlagsCopyLowering, correctly handle swapped and
immediate-adjusted flag reuse, and simplify EFLAGS live-in marking.
Adds MIR tests exercising multi-predecessor reuse over cyclic and
dominating CFGs, including a multi-level idom walk.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply at anthropic.com>
---
llvm/lib/Target/X86/X86FlagsCopyLowering.cpp | 11 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 171 ++++++-
llvm/lib/Target/X86/X86InstrInfo.h | 26 +
llvm/test/CodeGen/X86/apx/add.ll | 22 +-
.../CodeGen/X86/apx/flags-copy-lowering.ll | 4 +-
.../X86/apx/optimize-compare-multipred.ll | 483 ++++++++++++++++++
.../X86/apx/optimize-compare-multipred.mir | 183 +++++++
7 files changed, 858 insertions(+), 42 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/apx/optimize-compare-multipred.ll
create mode 100644 llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir
diff --git a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp
index 36b17cab03ee9..48c148f882456 100644
--- a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp
+++ b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp
@@ -68,8 +68,6 @@ STATISTIC(NumTestsInserted, "Number of test instructions inserted");
STATISTIC(NumAddsInserted, "Number of adds instructions inserted");
STATISTIC(NumNFsConvertedTo, "Number of NF instructions converted to");
-extern cl::opt<bool> X86EnableAPXForRelocation;
-
namespace {
// Convenient array type for storing registers associated with each condition.
@@ -254,14 +252,7 @@ static EFLAGSClobber getClobberType(const MachineInstr &MI) {
if (!FlagDef)
return NoClobber;
- // For the instructions are ADDrm/ADDmr with relocation, we'll skip the
- // optimization for replacing non-NF with NF. This is to keep backward
- // compatiblity with old version of linkers without APX relocation type
- // support on Linux OS.
- bool IsWithReloc =
- X86EnableAPXForRelocation ? false : isAddMemInstrWithRelocation(MI);
-
- if (FlagDef->isDead() && X86::getNFVariant(MI.getOpcode()) && !IsWithReloc)
+ if (X86::getNFVariantIfClobberRemovable(MI))
return EvitableClobber;
return InevitableClobber;
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 7ff2400d06d1d..e7410ddabee94 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -3290,6 +3290,21 @@ unsigned X86::getNFVariant(unsigned Opc) {
return getNewOpcFromTable(X86NFTransformTable, Opc);
}
+unsigned X86::getNFVariantIfClobberRemovable(const MachineInstr &MI,
+ const TargetRegisterInfo *TRI) {
+ if (!MI.registerDefIsDead(X86::EFLAGS, TRI))
+ return 0;
+ // For the instructions are ADDrm/ADDmr with relocation, we'll skip the
+ // optimization for replacing non-NF with NF. This is to keep backward
+ // compatiblity with old version of linkers without APX relocation type
+ // support on Linux OS.
+ bool IsWithReloc =
+ X86EnableAPXForRelocation ? false : isAddMemInstrWithRelocation(MI);
+ if (IsWithReloc)
+ return 0;
+ return X86::getNFVariant(MI.getOpcode());
+}
+
unsigned X86::getNonNDVariant(unsigned Opc) {
#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
// Make sure the tables are sorted.
@@ -5285,6 +5300,104 @@ static std::pair<X86::CondCode, unsigned> isUseDefConvertible(const MachineInstr
}
}
+MachineInstr *X86InstrInfo::findDominatingRedundantFlagInstr(
+ MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
+ int64_t CmpValue, MachineBasicBlock *MultiPredMBB, bool &IsSwapped,
+ int64_t &ImmDelta,
+ SmallVectorImpl<std::pair<MachineInstr *, unsigned>> &InstsToUpdate) const {
+ assert(Subtarget.hasNF() && "NF feature required");
+ MachineBasicBlock &CmpMBB = *CmpInstr.getParent();
+ MachineFunction &MF = *CmpMBB.getParent();
+ const TargetRegisterInfo *TRI = &getRegisterInfo();
+
+ // The caller already scanned MultiPredMBB (and any single-predecessor blocks
+ // between it and CmpMBB) without finding the producer, so the producer must
+ // live in a block that strictly dominates MultiPredMBB. Walk up the
+ // immediate-dominator chain. In each dominating block scan backward: the
+ // redundant producer ends the search; an NF-convertible clobber is stepped
+ // over (it will be collected by the path walk below) so the producer can
+ // still be found earlier in the same block or further up; any other EFLAGS
+ // clobber shadows the producer, so its flags cannot reach CmpInstr.
+ MachineDominatorTree MDT(MF);
+ MachineDomTreeNode *Node = MDT.getNode(MultiPredMBB);
+ MachineInstr *Sub = nullptr;
+ for (Node = Node ? Node->getIDom() : nullptr; Node && !Sub;
+ Node = Node->getIDom()) {
+ MachineBasicBlock *DomMBB = Node->getBlock();
+ for (MachineInstr &Inst : reverse(*DomMBB)) {
+ if (!Inst.modifiesRegister(X86::EFLAGS, TRI))
+ continue;
+ if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue,
+ Inst, &IsSwapped, &ImmDelta)) {
+ // Found the producer.
+ Sub = &Inst;
+ break;
+ }
+ if (!X86::getNFVariantIfClobberRemovable(Inst, TRI))
+ // A non-convertible clobber shadows any producer further up.
+ return nullptr;
+ // NF-convertible clobber: keep scanning earlier in this block.
+ }
+ // No producer in this block: keep walking up the dominator chain.
+ }
+ if (!Sub)
+ return nullptr;
+
+ // The forward condition-code fixup in the caller (OpsToUpdate) only rewrites
+ // EFLAGS users within CmpMBB. When the producer's flags require a condition
+ // swap or an immediate adjustment, EFLAGS users elsewhere in the dominated
+ // region or in CmpMBB's successors (when EFLAGS is live-out) would also need
+ // rewriting, which is not handled here. Restrict the multi-predecessor case
+ // to producers that yield identical flags.
+ if (IsSwapped || ImmDelta != 0)
+ return nullptr;
+ MachineBasicBlock *SubMBB = Sub->getParent();
+
+ // Verify that on every path from the producer to CmpInstr all EFLAGS clobbers
+ // are NF-convertible, collecting them. The caller already handled CmpMBB, the
+ // single-predecessor chain, and MultiPredMBB itself, so start from
+ // MultiPredMBB's predecessors and walk the CFG backward up to (and including
+ // the post-producer range of) SubMBB. Because SubMBB dominates MultiPredMBB,
+ // every block that can reach MultiPredMBB is dominated by SubMBB, so the walk
+ // reaches SubMBB on every path and never escapes above it.
+ //
+ // The walk never revisits a block (Visited) and never reaches a block the
+ // caller already scanned: MultiPredMBB is seeded into Visited, the
+ // single-predecessor chain blocks (CmpMBB..MultiPredMBB) each have exactly one
+ // predecessor by construction (the caller only advanced through such blocks),
+ // so none can be re-entered from the dominated region, and CmpMBB lies in the
+ // successor direction. No instruction is therefore collected twice.
+ SmallPtrSet<MachineBasicBlock *, 8> Visited;
+ SmallVector<MachineBasicBlock *, 8> Worklist;
+ Visited.insert(MultiPredMBB);
+ for (MachineBasicBlock *Pred : MultiPredMBB->predecessors())
+ if (Visited.insert(Pred).second)
+ Worklist.push_back(Pred);
+ while (!Worklist.empty()) {
+ MachineBasicBlock *MBB = Worklist.pop_back_val();
+ // For SubMBB only the range after the producer is relevant.
+ MachineBasicBlock::iterator I =
+ (MBB == SubMBB) ? std::next(MachineBasicBlock::iterator(Sub))
+ : MBB->begin();
+ for (MachineBasicBlock::iterator E = MBB->end(); I != E; ++I) {
+ MachineInstr &MI = *I;
+ if (!MI.modifiesRegister(X86::EFLAGS, TRI))
+ continue;
+ unsigned NewOpc = X86::getNFVariantIfClobberRemovable(MI, TRI);
+ if (!NewOpc)
+ return nullptr;
+ InstsToUpdate.push_back(std::make_pair(&MI, NewOpc));
+ }
+ if (MBB == SubMBB)
+ continue;
+ for (MachineBasicBlock *Pred : MBB->predecessors())
+ if (Visited.insert(Pred).second)
+ Worklist.push_back(Pred);
+ }
+
+ return Sub;
+}
+
/// Check if there exists an earlier instruction that
/// operates on the same source operands and sets flags in the same way as
/// Compare; remove Compare if possible.
@@ -5448,17 +5561,9 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
continue;
}
- // For the instructions are ADDrm/ADDmr with relocation, we'll skip the
- // optimization for replacing non-NF with NF. This is to keep backward
- // compatiblity with old version of linkers without APX relocation type
- // support on Linux OS.
- bool IsWithReloc = X86EnableAPXForRelocation
- ? false
- : isAddMemInstrWithRelocation(Inst);
-
// Try to replace non-NF with NF instructions.
- if (HasNF && Inst.registerDefIsDead(X86::EFLAGS, TRI) && !IsWithReloc) {
- unsigned NewOp = X86::getNFVariant(Inst.getOpcode());
+ if (HasNF) {
+ unsigned NewOp = X86::getNFVariantIfClobberRemovable(Inst, TRI);
if (!NewOp)
return false;
@@ -5474,10 +5579,28 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
if (MI || Sub)
break;
- // Reached begin of basic block. Continue in predecessor if there is
- // exactly one.
- if (MBB->pred_size() != 1)
- return false;
+ // Reached the begin of the basic block. If it has exactly one predecessor,
+ // continue the backward scan there. Otherwise (multiple predecessors), try
+ // to reuse EFLAGS from a dominating producer (handled below).
+ if (MBB->pred_size() != 1) {
+ // The block has multiple predecessors. We can still reuse EFLAGS from an
+ // equivalent flag producer that dominates CmpInstr, provided every path
+ // from that producer to CmpInstr only clobbers EFLAGS via instructions
+ // that have an NF (no-flags) variant (which requires APX). This handles
+ // patterns like (CMP duplicated by CodeGenPrepare across a diamond):
+ // entry: cmp %x, C ; br
+ // bb1: imul ... ; clobbers EFLAGS -> {nf} imul
+ // bb2: ...
+ // bb3: cmp %x, C ; <-- redundant, reuse EFLAGS from entry
+ // cmovcc ...
+ if (HasNF)
+ Sub = findDominatingRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2,
+ CmpMask, CmpValue, MBB, IsSwapped,
+ ImmDelta, InstsToUpdate);
+ if (!Sub)
+ return false;
+ break;
+ }
MBB = *MBB->pred_begin();
From = MBB->rbegin();
}
@@ -5683,11 +5806,25 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
.setImm(Op.second);
}
// Add EFLAGS to block live-ins between CmpBB and block of flags producer.
- for (MachineBasicBlock *MBB = &CmpMBB; MBB != SubBB;
- MBB = *MBB->pred_begin()) {
- assert(MBB->pred_size() == 1 && "Expected exactly one predecessor");
+ // Walk the CFG backward from CmpMBB up to (but excluding) SubBB, marking
+ // EFLAGS live-in on every block in between. SubBB dominates CmpMBB (whether
+ // the producer was found by the single-predecessor backward walk or the
+ // multi-predecessor dominator search), so the walk reaches SubBB on every
+ // path and never escapes above it. A single-predecessor chain is just the
+ // degenerate case where every block has exactly one predecessor.
+ SmallPtrSet<MachineBasicBlock *, 8> Visited;
+ SmallVector<MachineBasicBlock *, 8> Worklist(1, &CmpMBB);
+ Visited.insert(&CmpMBB);
+ while (!Worklist.empty()) {
+ MachineBasicBlock *MBB = Worklist.pop_back_val();
+ // EFLAGS is produced inside SubBB, so it is not live-in there.
+ if (MBB == SubBB)
+ continue;
if (!MBB->isLiveIn(X86::EFLAGS))
MBB->addLiveIn(X86::EFLAGS);
+ for (MachineBasicBlock *Pred : MBB->predecessors())
+ if (Visited.insert(Pred).second)
+ Worklist.push_back(Pred);
}
return true;
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index 2db0731da3c56..df0c74e1c7b9e 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -83,6 +83,14 @@ int getCCMPCondFlagsFromCondCode(CondCode CC);
// Get the opcode of corresponding NF variant.
unsigned getNFVariant(unsigned Opc);
+// If \p MI clobbers EFLAGS and that clobber can be removed by rewriting the
+// instruction to its NF (no-flags) variant, return that NF opcode; otherwise
+// return 0. The clobber is removable only if the EFLAGS def is dead, an NF
+// variant exists, and (for linker backward-compat) it is not an ADDrm/ADDmr
+// with relocation.
+unsigned getNFVariantIfClobberRemovable(const MachineInstr &MI,
+ const TargetRegisterInfo *TRI = nullptr);
+
// Get the opcode of corresponding NonND variant.
unsigned getNonNDVariant(unsigned Opc);
@@ -765,6 +773,24 @@ class X86InstrInfo final : public X86GenInstrInfo {
const MachineInstr &OI, bool *IsSwapped,
int64_t *ImmDelta) const;
+ /// Used by optimizeCompareInstr when the backward search for an equivalent
+ /// flag producer reaches a block (\p MultiPredMBB) with multiple
+ /// predecessors. Searches the dominator tree for an instruction that produces
+ /// the same flags as the compare \p CmpInstr and dominates it, such that on
+ /// every path from that producer to \p CmpInstr all EFLAGS clobbers can be
+ /// converted to their NF (no-flags) variants. Those clobbers are appended to
+ /// \p InstsToUpdate as (instruction, NF opcode) pairs for the caller to
+ /// rewrite. Restricted to producers that yield identical flags: it succeeds
+ /// only when isRedundantFlagInstr reports no operand swap and no immediate
+ /// delta, so on success \p IsSwapped is false and \p ImmDelta is 0. Returns
+ /// the flag producer on success, or nullptr otherwise. Requires the NF
+ /// feature (APX).
+ MachineInstr *findDominatingRedundantFlagInstr(
+ MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2,
+ int64_t CmpMask, int64_t CmpValue, MachineBasicBlock *MultiPredMBB,
+ bool &IsSwapped, int64_t &ImmDelta,
+ SmallVectorImpl<std::pair<MachineInstr *, unsigned>> &InstsToUpdate) const;
+
/// Commute operands of \p MI for memory fold.
///
/// \param Idx1 the index of operand to be commuted.
diff --git a/llvm/test/CodeGen/X86/apx/add.ll b/llvm/test/CodeGen/X86/apx/add.ll
index d7c5635b617c1..8b1e1ceb832a3 100644
--- a/llvm/test/CodeGen/X86/apx/add.ll
+++ b/llvm/test/CodeGen/X86/apx/add.ll
@@ -1221,26 +1221,24 @@ define i32 @two_address_no_subreg(i32 %arg0, ptr %arg1, i1 %arg2) nounwind {
; NF-NEXT: jne .LBB50_2 # encoding: [0x75,A]
; NF-NEXT: # fixup A - offset: 1, value: .LBB50_2, kind: FK_PCRel_1
; NF-NEXT: # %bb.1: # %bb2
-; NF-NEXT: xorl %r12d, %r12d # encoding: [0x45,0x31,0xe4]
+; NF-NEXT: movl $0, %r12d # encoding: [0x41,0xbc,0x00,0x00,0x00,0x00]
; NF-NEXT: .LBB50_2: # %bb1
-; NF-NEXT: movl %r13d, %esi # encoding: [0x44,0x89,0xee]
-; NF-NEXT: addl (%rsp), %esi # 4-byte Folded Reload
-; NF-NEXT: # encoding: [0x03,0x34,0x24]
-; NF-NEXT: {nf} imull %ebx, %ebx, %r13d # EVEX TO EVEX Compression encoding: [0x62,0xf4,0x14,0x1c,0xaf,0xdb]
-; NF-NEXT: testb $1, %bpl # encoding: [0x40,0xf6,0xc5,0x01]
+; NF-NEXT: movq (%rsp), %rax # 8-byte Reload
+; NF-NEXT: # encoding: [0x48,0x8b,0x04,0x24]
+; NF-NEXT: {nf} addl %eax, %r13d, %ebp # encoding: [0x62,0xd4,0x54,0x1c,0x01,0xc5]
+; NF-NEXT: {nf} imull %ebx, %ebx, %r13d # encoding: [0x62,0xf4,0x14,0x1c,0xaf,0xdb]
; NF-NEXT: jne .LBB50_4 # encoding: [0x75,A]
; NF-NEXT: # fixup A - offset: 1, value: .LBB50_4, kind: FK_PCRel_1
; NF-NEXT: # %bb.3: # %bb4
-; NF-NEXT: xorl %ebp, %ebp # encoding: [0x31,0xed]
+; NF-NEXT: xorl %eax, %eax # encoding: [0x31,0xc0]
; NF-NEXT: movq %r14, %rdi # encoding: [0x4c,0x89,0xf7]
-; NF-NEXT: movl %esi, %r14d # encoding: [0x41,0x89,0xf6]
-; NF-NEXT: callq *%rbp # encoding: [0xff,0xd5]
+; NF-NEXT: callq *%rax # encoding: [0xff,0xd0]
+; NF-NEXT: xorl %eax, %eax # encoding: [0x31,0xc0]
; NF-NEXT: {nf} incl %r12d, %r15d # EVEX TO EVEX Compression encoding: [0x62,0xd4,0x04,0x1c,0xff,0xc4]
; NF-NEXT: xorl %edi, %edi # encoding: [0x31,0xff]
-; NF-NEXT: callq *%rbp # encoding: [0xff,0xd5]
-; NF-NEXT: movl %r14d, %esi # encoding: [0x44,0x89,0xf6]
+; NF-NEXT: callq *%rax # encoding: [0xff,0xd0]
; NF-NEXT: .LBB50_4: # %bb3
-; NF-NEXT: orl %r13d, %esi # EVEX TO LEGACY Compression encoding: [0x44,0x09,0xee]
+; NF-NEXT: {nf} orl %r13d, %ebp, %esi # EVEX TO EVEX Compression encoding: [0x62,0x74,0x4c,0x1c,0x09,0xed]
; NF-NEXT: orl %r15d, %esi # EVEX TO LEGACY Compression encoding: [0x44,0x09,0xfe]
; NF-NEXT: xorl %eax, %eax # encoding: [0x31,0xc0]
; NF-NEXT: movl %ebx, %edi # encoding: [0x89,0xdf]
diff --git a/llvm/test/CodeGen/X86/apx/flags-copy-lowering.ll b/llvm/test/CodeGen/X86/apx/flags-copy-lowering.ll
index 0dcda8efdbc78..357126bff417d 100644
--- a/llvm/test/CodeGen/X86/apx/flags-copy-lowering.ll
+++ b/llvm/test/CodeGen/X86/apx/flags-copy-lowering.ll
@@ -51,7 +51,6 @@ define <2 x i128> @flag_copy_2(<2 x i128> %x, <2 x i128> %y) nounwind {
ret <2 x i128> %z
}
-; TODO: Remove the 2nd cmpl by using NF imul.
define void @flag_copy_3(i32 %x, i32 %y, ptr %pa, ptr %pb, ptr %pc) nounwind {
; CHECK-LABEL: flag_copy_3:
; CHECK: # %bb.0: # %entry
@@ -60,14 +59,13 @@ define void @flag_copy_3(i32 %x, i32 %y, ptr %pa, ptr %pb, ptr %pc) nounwind {
; CHECK-NEXT: jl .LBB2_2
; CHECK-NEXT: # %bb.1: # %bb1
; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: imull %esi, %eax
+; CHECK-NEXT: {nf} imull %esi, %eax
; CHECK-NEXT: movl %eax, (%rdx)
; CHECK-NEXT: jmp .LBB2_3
; CHECK-NEXT: .LBB2_2: # %bb2
; CHECK-NEXT: leal -2(%rsi), %eax
; CHECK-NEXT: movl %eax, (%rcx)
; CHECK-NEXT: .LBB2_3: # %bb3
-; CHECK-NEXT: cmpl $2, %edi
; CHECK-NEXT: cmovgel %edi, %esi
; CHECK-NEXT: movl %esi, (%r8)
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.ll b/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.ll
new file mode 100644
index 0000000000000..0cf2469f78ba3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.ll
@@ -0,0 +1,483 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+nf | FileCheck %s --check-prefixes=NF
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=NONF
+
+; Tests for optimizeCompareInstr reusing EFLAGS across a block with multiple
+; predecessors via findDominatingRedundantFlagInstr (requires APX NF feature).
+; A compare duplicated by CodeGenPrepare across a diamond can be removed when
+; every path from the dominating producer to the redundant compare only
+; clobbers EFLAGS through NF-convertible instructions.
+
+declare void @ext()
+ at g = external global i32
+
+; Positive: the entry compare dominates the redundant compare in bb3; the only
+; EFLAGS clobber on the path (imul on bb1) is NF-convertible, so the second
+; compare is removed and the imul becomes {nf} imul.
+define void @diamond_one_arm(i32 %x, i32 %y, ptr %pa, ptr %pb, ptr %pc) nounwind {
+; NF-LABEL: diamond_one_arm:
+; NF: # %bb.0: # %entry
+; NF-NEXT: # kill: def $esi killed $esi def $rsi
+; NF-NEXT: cmpl $2, %edi
+; NF-NEXT: jl .LBB0_2
+; NF-NEXT: # %bb.1: # %bb1
+; NF-NEXT: movl %edi, %eax
+; NF-NEXT: {nf} imull %esi, %eax
+; NF-NEXT: movl %eax, (%rdx)
+; NF-NEXT: jmp .LBB0_3
+; NF-NEXT: .LBB0_2: # %bb2
+; NF-NEXT: leal -2(%rsi), %eax
+; NF-NEXT: movl %eax, (%rcx)
+; NF-NEXT: .LBB0_3: # %bb3
+; NF-NEXT: cmovgel %edi, %esi
+; NF-NEXT: movl %esi, (%r8)
+; NF-NEXT: retq
+;
+; NONF-LABEL: diamond_one_arm:
+; NONF: # %bb.0: # %entry
+; NONF-NEXT: # kill: def $esi killed $esi def $rsi
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: jl .LBB0_2
+; NONF-NEXT: # %bb.1: # %bb1
+; NONF-NEXT: movl %edi, %eax
+; NONF-NEXT: imull %esi, %eax
+; NONF-NEXT: movl %eax, (%rdx)
+; NONF-NEXT: jmp .LBB0_3
+; NONF-NEXT: .LBB0_2: # %bb2
+; NONF-NEXT: leal -2(%rsi), %eax
+; NONF-NEXT: movl %eax, (%rcx)
+; NONF-NEXT: .LBB0_3: # %bb3
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: cmovgel %edi, %esi
+; NONF-NEXT: movl %esi, (%r8)
+; NONF-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 1
+ br i1 %cmp, label %bb1, label %bb2
+bb1:
+ %mul = mul nuw nsw i32 %x, %y
+ store i32 %mul, ptr %pa
+ br label %bb3
+bb2:
+ %sub = sub nuw nsw i32 %y, 2
+ store i32 %sub, ptr %pb
+ br label %bb3
+bb3:
+ %s = select i1 %cmp, i32 %x, i32 %y
+ store i32 %s, ptr %pc
+ ret void
+}
+
+; Positive: NF-convertible EFLAGS clobbers on BOTH diamond arms. The backward
+; CFG walk collects both, converts both to {nf}, and removes the redundant
+; compare.
+define void @diamond_both_arms(i32 %x, i32 %y, ptr %pa, ptr %pb, ptr %pc) nounwind {
+; NF-LABEL: diamond_both_arms:
+; NF: # %bb.0: # %entry
+; NF-NEXT: cmpl $2, %edi
+; NF-NEXT: jl .LBB1_2
+; NF-NEXT: # %bb.1: # %bb1
+; NF-NEXT: movl %edi, %eax
+; NF-NEXT: {nf} imull %esi, %eax
+; NF-NEXT: movl %eax, (%rdx)
+; NF-NEXT: jmp .LBB1_3
+; NF-NEXT: .LBB1_2: # %bb2
+; NF-NEXT: movl %esi, %eax
+; NF-NEXT: {nf} imull %esi, %eax
+; NF-NEXT: movl %eax, (%rcx)
+; NF-NEXT: .LBB1_3: # %bb3
+; NF-NEXT: cmovgel %edi, %esi
+; NF-NEXT: movl %esi, (%r8)
+; NF-NEXT: retq
+;
+; NONF-LABEL: diamond_both_arms:
+; NONF: # %bb.0: # %entry
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: jl .LBB1_2
+; NONF-NEXT: # %bb.1: # %bb1
+; NONF-NEXT: movl %edi, %eax
+; NONF-NEXT: imull %esi, %eax
+; NONF-NEXT: movl %eax, (%rdx)
+; NONF-NEXT: jmp .LBB1_3
+; NONF-NEXT: .LBB1_2: # %bb2
+; NONF-NEXT: movl %esi, %eax
+; NONF-NEXT: imull %esi, %eax
+; NONF-NEXT: movl %eax, (%rcx)
+; NONF-NEXT: .LBB1_3: # %bb3
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: cmovgel %edi, %esi
+; NONF-NEXT: movl %esi, (%r8)
+; NONF-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 1
+ br i1 %cmp, label %bb1, label %bb2
+bb1:
+ %m1 = mul nuw nsw i32 %x, %y
+ store i32 %m1, ptr %pa
+ br label %bb3
+bb2:
+ %m2 = mul nuw nsw i32 %y, %y
+ store i32 %m2, ptr %pb
+ br label %bb3
+bb3:
+ %s = select i1 %cmp, i32 %x, i32 %y
+ store i32 %s, ptr %pc
+ ret void
+}
+
+; Positive: the producer dominates the multi-predecessor block %join several
+; levels up (idom chain %join -> %pre -> %entry), and every EFLAGS clobber on
+; the paths is an NF-convertible imul. Both redundant compares are removed: the
+; one in %pre by the single-predecessor backward walk (its only predecessor is
+; %entry), and the one in %join by the multi-predecessor dominator search.
+define void @nested_dominator(i32 %x, i32 %y, ptr %pa, ptr %pb, ptr %pc, ptr %pd) nounwind {
+; NF-LABEL: nested_dominator:
+; NF: # %bb.0: # %entry
+; NF-NEXT: cmpl $2, %edi
+; NF-NEXT: jl .LBB2_5
+; NF-NEXT: # %bb.1: # %pre
+; NF-NEXT: movl %edi, %eax
+; NF-NEXT: {nf} imull %esi, %eax
+; NF-NEXT: movl %eax, (%rdx)
+; NF-NEXT: jl .LBB2_3
+; NF-NEXT: # %bb.2: # %bb1
+; NF-NEXT: movl %edi, %eax
+; NF-NEXT: {nf} imull %edi, %eax
+; NF-NEXT: movl %eax, (%rcx)
+; NF-NEXT: jmp .LBB2_4
+; NF-NEXT: .LBB2_3: # %bb2
+; NF-NEXT: movl %esi, %eax
+; NF-NEXT: {nf} imull %esi, %eax
+; NF-NEXT: movl %eax, (%r8)
+; NF-NEXT: .LBB2_4: # %join
+; NF-NEXT: cmovgel %edi, %esi
+; NF-NEXT: movl %esi, (%r9)
+; NF-NEXT: .LBB2_5: # %ret
+; NF-NEXT: retq
+;
+; NONF-LABEL: nested_dominator:
+; NONF: # %bb.0: # %entry
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: jl .LBB2_5
+; NONF-NEXT: # %bb.1: # %pre
+; NONF-NEXT: movl %edi, %eax
+; NONF-NEXT: imull %esi, %eax
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: movl %eax, (%rdx)
+; NONF-NEXT: jl .LBB2_3
+; NONF-NEXT: # %bb.2: # %bb1
+; NONF-NEXT: movl %edi, %eax
+; NONF-NEXT: imull %edi, %eax
+; NONF-NEXT: movl %eax, (%rcx)
+; NONF-NEXT: jmp .LBB2_4
+; NONF-NEXT: .LBB2_3: # %bb2
+; NONF-NEXT: movl %esi, %eax
+; NONF-NEXT: imull %esi, %eax
+; NONF-NEXT: movl %eax, (%r8)
+; NONF-NEXT: .LBB2_4: # %join
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: cmovgel %edi, %esi
+; NONF-NEXT: movl %esi, (%r9)
+; NONF-NEXT: .LBB2_5: # %ret
+; NONF-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 1
+ br i1 %cmp, label %pre, label %ret
+pre:
+ %m0 = mul nuw nsw i32 %x, %y
+ store i32 %m0, ptr %pa
+ br i1 %cmp, label %bb1, label %bb2
+bb1:
+ %m1 = mul nuw nsw i32 %x, %x
+ store i32 %m1, ptr %pb
+ br label %join
+bb2:
+ %m2 = mul nuw nsw i32 %y, %y
+ store i32 %m2, ptr %pc
+ br label %join
+join:
+ %s = select i1 %cmp, i32 %x, i32 %y
+ store i32 %s, ptr %pd
+ br label %ret
+ret:
+ ret void
+}
+
+; Negative: a call clobbers EFLAGS on the path (regmask clobber, no NF variant),
+; so the redundant compare must be kept.
+define void @call_on_path(i32 %x, i32 %y, ptr %pa, ptr %pc) nounwind {
+; NF-LABEL: call_on_path:
+; NF: # %bb.0: # %entry
+; NF-NEXT: pushq %rbp
+; NF-NEXT: pushq %r15
+; NF-NEXT: pushq %r14
+; NF-NEXT: pushq %rbx
+; NF-NEXT: pushq %rax
+; NF-NEXT: movq %rcx, %rbx
+; NF-NEXT: movl %esi, %ebp
+; NF-NEXT: movl %edi, %r14d
+; NF-NEXT: cmpl $2, %edi
+; NF-NEXT: jl .LBB3_2
+; NF-NEXT: # %bb.1: # %bb1
+; NF-NEXT: movq %rdx, %r15
+; NF-NEXT: callq ext at PLT
+; NF-NEXT: movl %ebp, (%r15)
+; NF-NEXT: .LBB3_2: # %bb3
+; NF-NEXT: cmpl $2, %r14d
+; NF-NEXT: cmovgel %r14d, %ebp
+; NF-NEXT: movl %ebp, (%rbx)
+; NF-NEXT: addq $8, %rsp
+; NF-NEXT: popq %rbx
+; NF-NEXT: popq %r14
+; NF-NEXT: popq %r15
+; NF-NEXT: popq %rbp
+; NF-NEXT: retq
+;
+; NONF-LABEL: call_on_path:
+; NONF: # %bb.0: # %entry
+; NONF-NEXT: pushq %rbp
+; NONF-NEXT: pushq %r15
+; NONF-NEXT: pushq %r14
+; NONF-NEXT: pushq %rbx
+; NONF-NEXT: pushq %rax
+; NONF-NEXT: movq %rcx, %rbx
+; NONF-NEXT: movl %esi, %ebp
+; NONF-NEXT: movl %edi, %r14d
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: jl .LBB3_2
+; NONF-NEXT: # %bb.1: # %bb1
+; NONF-NEXT: movq %rdx, %r15
+; NONF-NEXT: callq ext at PLT
+; NONF-NEXT: movl %ebp, (%r15)
+; NONF-NEXT: .LBB3_2: # %bb3
+; NONF-NEXT: cmpl $2, %r14d
+; NONF-NEXT: cmovgel %r14d, %ebp
+; NONF-NEXT: movl %ebp, (%rbx)
+; NONF-NEXT: addq $8, %rsp
+; NONF-NEXT: popq %rbx
+; NONF-NEXT: popq %r14
+; NONF-NEXT: popq %r15
+; NONF-NEXT: popq %rbp
+; NONF-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 1
+ br i1 %cmp, label %bb1, label %bb2
+bb1:
+ call void @ext()
+ store i32 %y, ptr %pa
+ br label %bb3
+bb2:
+ br label %bb3
+bb3:
+ %s = select i1 %cmp, i32 %x, i32 %y
+ store i32 %s, ptr %pc
+ ret void
+}
+
+; Negative: a flag-consuming compare/branch (testl + je) sits on the path; its
+; EFLAGS def is not dead, so it is not NF-convertible and the redundant compare
+; is kept.
+define void @flag_user_on_path(i32 %x, i32 %y, ptr %pa, ptr %pb, ptr %pc) nounwind {
+; NF-LABEL: flag_user_on_path:
+; NF: # %bb.0: # %entry
+; NF-NEXT: cmpl $2, %edi
+; NF-NEXT: jl .LBB4_3
+; NF-NEXT: # %bb.1: # %bb1
+; NF-NEXT: testl %esi, %esi
+; NF-NEXT: je .LBB4_3
+; NF-NEXT: # %bb.2: # %bb2
+; NF-NEXT: movl %esi, (%rcx)
+; NF-NEXT: .LBB4_3: # %bb3
+; NF-NEXT: cmpl $2, %edi
+; NF-NEXT: cmovgel %edi, %esi
+; NF-NEXT: movl %esi, (%r8)
+; NF-NEXT: retq
+;
+; NONF-LABEL: flag_user_on_path:
+; NONF: # %bb.0: # %entry
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: jl .LBB4_3
+; NONF-NEXT: # %bb.1: # %bb1
+; NONF-NEXT: testl %esi, %esi
+; NONF-NEXT: je .LBB4_3
+; NONF-NEXT: # %bb.2: # %bb2
+; NONF-NEXT: movl %esi, (%rcx)
+; NONF-NEXT: .LBB4_3: # %bb3
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: cmovgel %edi, %esi
+; NONF-NEXT: movl %esi, (%r8)
+; NONF-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 1
+ br i1 %cmp, label %bb1, label %bb3
+bb1:
+ %nz = icmp ne i32 %y, 0
+ br i1 %nz, label %bb2, label %bb3
+bb2:
+ store i32 %y, ptr %pb
+ br label %bb3
+bb3:
+ %s = select i1 %cmp, i32 %x, i32 %y
+ store i32 %s, ptr %pc
+ ret void
+}
+
+; Negative: the redundant compare uses swapped operands (cmp y, x vs cmp x, y).
+; The multi-predecessor path is restricted to identical flags, so it bails.
+define void @swapped_operands(i32 %x, i32 %y, ptr %pa, ptr %pb, ptr %pc) nounwind {
+; NF-LABEL: swapped_operands:
+; NF: # %bb.0: # %entry
+; NF-NEXT: cmpl %esi, %edi
+; NF-NEXT: jle .LBB5_2
+; NF-NEXT: # %bb.1: # %bb1
+; NF-NEXT: movl %edi, %eax
+; NF-NEXT: imull %esi, %eax
+; NF-NEXT: movl %eax, (%rdx)
+; NF-NEXT: jmp .LBB5_3
+; NF-NEXT: .LBB5_2: # %bb2
+; NF-NEXT: movl %esi, (%rcx)
+; NF-NEXT: .LBB5_3: # %bb3
+; NF-NEXT: cmpl %edi, %esi
+; NF-NEXT: cmovll %edi, %esi
+; NF-NEXT: movl %esi, (%r8)
+; NF-NEXT: retq
+;
+; NONF-LABEL: swapped_operands:
+; NONF: # %bb.0: # %entry
+; NONF-NEXT: cmpl %esi, %edi
+; NONF-NEXT: jle .LBB5_2
+; NONF-NEXT: # %bb.1: # %bb1
+; NONF-NEXT: movl %edi, %eax
+; NONF-NEXT: imull %esi, %eax
+; NONF-NEXT: movl %eax, (%rdx)
+; NONF-NEXT: jmp .LBB5_3
+; NONF-NEXT: .LBB5_2: # %bb2
+; NONF-NEXT: movl %esi, (%rcx)
+; NONF-NEXT: .LBB5_3: # %bb3
+; NONF-NEXT: cmpl %edi, %esi
+; NONF-NEXT: cmovll %edi, %esi
+; NONF-NEXT: movl %esi, (%r8)
+; NONF-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, %y
+ br i1 %cmp, label %bb1, label %bb2
+bb1:
+ %mul = mul nuw nsw i32 %x, %y
+ store i32 %mul, ptr %pa
+ br label %bb3
+bb2:
+ store i32 %y, ptr %pb
+ br label %bb3
+bb3:
+ %cmp2 = icmp slt i32 %y, %x
+ %s = select i1 %cmp2, i32 %x, i32 %y
+ store i32 %s, ptr %pc
+ ret void
+}
+
+; Negative: the redundant compare differs by an immediate delta (x>1 vs x>2),
+; which would require a condition-code adjustment; the multi-predecessor path
+; bails.
+define void @imm_delta(i32 %x, i32 %y, ptr %pa, ptr %pb, ptr %pc) nounwind {
+; NF-LABEL: imm_delta:
+; NF: # %bb.0: # %entry
+; NF-NEXT: cmpl $2, %edi
+; NF-NEXT: jl .LBB6_2
+; NF-NEXT: # %bb.1: # %bb1
+; NF-NEXT: movl %edi, %eax
+; NF-NEXT: imull %esi, %eax
+; NF-NEXT: movl %eax, (%rdx)
+; NF-NEXT: jmp .LBB6_3
+; NF-NEXT: .LBB6_2: # %bb2
+; NF-NEXT: movl %esi, (%rcx)
+; NF-NEXT: .LBB6_3: # %bb3
+; NF-NEXT: cmpl $3, %edi
+; NF-NEXT: cmovgel %edi, %esi
+; NF-NEXT: movl %esi, (%r8)
+; NF-NEXT: retq
+;
+; NONF-LABEL: imm_delta:
+; NONF: # %bb.0: # %entry
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: jl .LBB6_2
+; NONF-NEXT: # %bb.1: # %bb1
+; NONF-NEXT: movl %edi, %eax
+; NONF-NEXT: imull %esi, %eax
+; NONF-NEXT: movl %eax, (%rdx)
+; NONF-NEXT: jmp .LBB6_3
+; NONF-NEXT: .LBB6_2: # %bb2
+; NONF-NEXT: movl %esi, (%rcx)
+; NONF-NEXT: .LBB6_3: # %bb3
+; NONF-NEXT: cmpl $3, %edi
+; NONF-NEXT: cmovgel %edi, %esi
+; NONF-NEXT: movl %esi, (%r8)
+; NONF-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 1
+ br i1 %cmp, label %bb1, label %bb2
+bb1:
+ %mul = mul nuw nsw i32 %x, %y
+ store i32 %mul, ptr %pa
+ br label %bb3
+bb2:
+ store i32 %y, ptr %pb
+ br label %bb3
+bb3:
+ %cmp2 = icmp sgt i32 %x, 2
+ %s = select i1 %cmp2, i32 %x, i32 %y
+ store i32 %s, ptr %pc
+ ret void
+}
+
+; Negative: no dominating producer compares the same value, so there is nothing
+; to reuse and the compare is kept.
+define void @no_producer(i32 %x, i32 %y, i32 %z, ptr %pa, ptr %pb, ptr %pc) nounwind {
+; NF-LABEL: no_producer:
+; NF: # %bb.0: # %entry
+; NF-NEXT: cmpl $2, %edi
+; NF-NEXT: jl .LBB7_2
+; NF-NEXT: # %bb.1: # %bb1
+; NF-NEXT: movl %edi, %eax
+; NF-NEXT: imull %esi, %eax
+; NF-NEXT: movl %eax, (%rcx)
+; NF-NEXT: jmp .LBB7_3
+; NF-NEXT: .LBB7_2: # %bb2
+; NF-NEXT: movl %esi, (%r8)
+; NF-NEXT: .LBB7_3: # %bb3
+; NF-NEXT: cmpl $6, %edx
+; NF-NEXT: cmovgel %edi, %esi
+; NF-NEXT: movl %esi, (%r9)
+; NF-NEXT: retq
+;
+; NONF-LABEL: no_producer:
+; NONF: # %bb.0: # %entry
+; NONF-NEXT: cmpl $2, %edi
+; NONF-NEXT: jl .LBB7_2
+; NONF-NEXT: # %bb.1: # %bb1
+; NONF-NEXT: movl %edi, %eax
+; NONF-NEXT: imull %esi, %eax
+; NONF-NEXT: movl %eax, (%rcx)
+; NONF-NEXT: jmp .LBB7_3
+; NONF-NEXT: .LBB7_2: # %bb2
+; NONF-NEXT: movl %esi, (%r8)
+; NONF-NEXT: .LBB7_3: # %bb3
+; NONF-NEXT: cmpl $6, %edx
+; NONF-NEXT: cmovgel %edi, %esi
+; NONF-NEXT: movl %esi, (%r9)
+; NONF-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 1
+ br i1 %cmp, label %bb1, label %bb2
+bb1:
+ %mul = mul nuw nsw i32 %x, %y
+ store i32 %mul, ptr %pa
+ br label %bb3
+bb2:
+ store i32 %y, ptr %pb
+ br label %bb3
+bb3:
+ %cmp2 = icmp sgt i32 %z, 5
+ %s = select i1 %cmp2, i32 %x, i32 %y
+ store i32 %s, ptr %pc
+ ret void
+}
diff --git a/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir b/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir
new file mode 100644
index 0000000000000..a4af76efe8452
--- /dev/null
+++ b/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir
@@ -0,0 +1,183 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -o - %s -mtriple=x86_64-- -run-pass peephole-opt -mattr=+nf | FileCheck %s
+
+# MIR-level coverage for the multi-predecessor EFLAGS reuse in
+# optimizeCompareInstr / findDominatingRedundantFlagInstr. These CFG shapes are
+# awkward to produce from IR (LICM hoists loop-invariant compares, the scheduler
+# repositions EFLAGS clobbers), so they are written directly as MIR.
+
+# A non-NF-convertible EFLAGS clobber in the dominating block that defines the
+# flags shadows the producer, so the redundant compare in bb.3 must be kept.
+# (The clobber is rejected by both the idom-chain scan and the subsequent
+# path walk; either alone is sufficient to bail.)
+---
+name: dom_shadow_clobber
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: dom_shadow_clobber
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $edi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: CMP32ri [[COPY1]], 7, implicit-def $eflags
+ ; CHECK-NEXT: JCC_1 %bb.2, 12, implicit $eflags
+ ; CHECK-NEXT: JMP_1 %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[IMUL32rr:%[0-9]+]]:gr32 = nuw nsw IMUL32rr [[COPY]], [[COPY1]], implicit-def dead $eflags
+ ; CHECK-NEXT: JMP_1 %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: $al = SETCCr 15, implicit $eflags
+ ; CHECK-NEXT: RET 0, $al
+ bb.0:
+ liveins: $edi, $esi
+ %0:gr32 = COPY $edi
+ %1:gr32 = COPY $esi
+ CMP32ri %0, 2, implicit-def $eflags
+ CMP32ri %1, 7, implicit-def $eflags
+ JCC_1 %bb.2, 12, implicit $eflags
+ JMP_1 %bb.1
+
+ bb.1:
+ %2:gr32 = nuw nsw IMUL32rr %0, %1, implicit-def dead $eflags
+ JMP_1 %bb.3
+
+ bb.2:
+
+ bb.3:
+ CMP32ri %0, 2, implicit-def $eflags
+ $al = SETCCr 15, implicit $eflags
+ RET 0, $al
+...
+
+# Same shape WITHOUT the shadow clobber: the producer in the dominating block is
+# found, the redundant compare in bb.3 is removed and the IMUL on the path is
+# rewritten to its NF variant. Positive control for dom_shadow_clobber.
+---
+name: dom_no_shadow
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: dom_no_shadow
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $edi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: JCC_1 %bb.2, 12, implicit $eflags
+ ; CHECK-NEXT: JMP_1 %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[IMUL32rr_NF:%[0-9]+]]:gr32 = nuw nsw IMUL32rr_NF [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: JMP_1 %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: liveins: $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $al = SETCCr 15, implicit $eflags
+ ; CHECK-NEXT: RET 0, $al
+ bb.0:
+ liveins: $edi, $esi
+ %0:gr32 = COPY $edi
+ %1:gr32 = COPY $esi
+ CMP32ri %0, 2, implicit-def $eflags
+ JCC_1 %bb.2, 12, implicit $eflags
+ JMP_1 %bb.1
+
+ bb.1:
+ %2:gr32 = nuw nsw IMUL32rr %0, %1, implicit-def dead $eflags
+ JMP_1 %bb.3
+
+ bb.2:
+
+ bb.3:
+ CMP32ri %0, 2, implicit-def $eflags
+ $al = SETCCr 15, implicit $eflags
+ RET 0, $al
+...
+
+# Loop: the producer in bb.0 dominates the multi-predecessor block bb.1 (preds
+# bb.0 and the bb.3 back-edge). The redundant compare in bb.1 is removed and the
+# NF-convertible IMUL on the bb.2 path is rewritten. This exercises the
+# verification and live-in walks over a cyclic CFG with a back-edge (the Visited
+# sets keep both walks terminating), which the acyclic diamonds in
+# optimize-compare-multipred.ll do not cover.
+---
+name: loop_backedge
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: loop_backedge
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $edi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: JMP_1 %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: JCC_1 %bb.4, 4, implicit $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[IMUL32rr_NF:%[0-9]+]]:gr32 = nuw nsw IMUL32rr_NF [[COPY]], [[COPY1]]
+ ; CHECK-NEXT: JMP_1 %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: JMP_1 %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: liveins: $eflags
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $al = SETCCr 15, implicit $eflags
+ ; CHECK-NEXT: RET 0, $al
+ bb.0:
+ liveins: $edi, $esi
+ %0:gr32 = COPY $edi
+ %1:gr32 = COPY $esi
+ CMP32ri %0, 2, implicit-def $eflags
+ JMP_1 %bb.1
+
+ bb.1:
+ CMP32ri %0, 2, implicit-def $eflags
+ JCC_1 %bb.4, 4, implicit $eflags
+
+ bb.2:
+ %2:gr32 = nuw nsw IMUL32rr %0, %1, implicit-def dead $eflags
+ JMP_1 %bb.3
+
+ bb.3:
+ JMP_1 %bb.1
+
+ bb.4:
+ liveins: $eflags
+ $al = SETCCr 15, implicit $eflags
+ RET 0, $al
+...
>From 4bf5fd9ef26392eeaa0c2da5a4b0ed35b726ea7d Mon Sep 17 00:00:00 2001
From: Feng Zou <feng.zou at intel.com>
Date: Wed, 8 Jul 2026 20:44:57 +0800
Subject: [PATCH 2/4] apply clang format
---
llvm/lib/Target/X86/X86InstrInfo.cpp | 15 ++++++++-------
llvm/lib/Target/X86/X86InstrInfo.h | 8 +++++---
2 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index e7410ddabee94..03490671a3130 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -5363,10 +5363,11 @@ MachineInstr *X86InstrInfo::findDominatingRedundantFlagInstr(
//
// The walk never revisits a block (Visited) and never reaches a block the
// caller already scanned: MultiPredMBB is seeded into Visited, the
- // single-predecessor chain blocks (CmpMBB..MultiPredMBB) each have exactly one
- // predecessor by construction (the caller only advanced through such blocks),
- // so none can be re-entered from the dominated region, and CmpMBB lies in the
- // successor direction. No instruction is therefore collected twice.
+ // single-predecessor chain blocks (CmpMBB..MultiPredMBB) each have exactly
+ // one predecessor by construction (the caller only advanced through such
+ // blocks), so none can be re-entered from the dominated region, and CmpMBB
+ // lies in the successor direction. No instruction is therefore collected
+ // twice.
SmallPtrSet<MachineBasicBlock *, 8> Visited;
SmallVector<MachineBasicBlock *, 8> Worklist;
Visited.insert(MultiPredMBB);
@@ -5594,9 +5595,9 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
// bb3: cmp %x, C ; <-- redundant, reuse EFLAGS from entry
// cmovcc ...
if (HasNF)
- Sub = findDominatingRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2,
- CmpMask, CmpValue, MBB, IsSwapped,
- ImmDelta, InstsToUpdate);
+ Sub = findDominatingRedundantFlagInstr(
+ CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue, MBB, IsSwapped,
+ ImmDelta, InstsToUpdate);
if (!Sub)
return false;
break;
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index df0c74e1c7b9e..06f942d4fdd9c 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -88,8 +88,9 @@ unsigned getNFVariant(unsigned Opc);
// return 0. The clobber is removable only if the EFLAGS def is dead, an NF
// variant exists, and (for linker backward-compat) it is not an ADDrm/ADDmr
// with relocation.
-unsigned getNFVariantIfClobberRemovable(const MachineInstr &MI,
- const TargetRegisterInfo *TRI = nullptr);
+unsigned
+getNFVariantIfClobberRemovable(const MachineInstr &MI,
+ const TargetRegisterInfo *TRI = nullptr);
// Get the opcode of corresponding NonND variant.
unsigned getNonNDVariant(unsigned Opc);
@@ -789,7 +790,8 @@ class X86InstrInfo final : public X86GenInstrInfo {
MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2,
int64_t CmpMask, int64_t CmpValue, MachineBasicBlock *MultiPredMBB,
bool &IsSwapped, int64_t &ImmDelta,
- SmallVectorImpl<std::pair<MachineInstr *, unsigned>> &InstsToUpdate) const;
+ SmallVectorImpl<std::pair<MachineInstr *, unsigned>> &InstsToUpdate)
+ const;
/// Commute operands of \p MI for memory fold.
///
>From f262eb62299a282d7315e48f8901edb176b9f330 Mon Sep 17 00:00:00 2001
From: Feng Zou <feng.zou at intel.com>
Date: Fri, 10 Jul 2026 15:25:25 +0800
Subject: [PATCH 3/4] [X86] Address review comments on multi-predecessor EFLAGS
reuse
- Simplify the relocation guard in getNFVariantIfClobberRemovable to an early
return.
- Early-return in findDominatingRedundantFlagInstr when the multi-predecessor
block has no dominator-tree node.
- Add a profitability gate at the optimizeCompareInstr call site: only attempt
the dominating reuse when no NF conversion has been collected yet
(HasNF && InstsToUpdate.empty()). An already-collected conversion lies on the
single-predecessor chain, i.e. the unconditional path to CmpInstr that the
compare removal saves, so converting it is pure added cost (the
two_address_no_subreg case); conversions the dominating walk collects instead
lie on the mutually exclusive diamond arms and are only conditionally
executed. This reverts the add.ll churn back to baseline.
- Add a chain_conv_unprofitable MIR test covering the new guard.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply at anthropic.com>
---
llvm/lib/Target/X86/X86InstrInfo.cpp | 18 ++++--
llvm/test/CodeGen/X86/apx/add.ll | 22 ++++---
.../X86/apx/optimize-compare-multipred.mir | 64 +++++++++++++++++++
3 files changed, 88 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 03490671a3130..a598437789f7f 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -3298,9 +3298,7 @@ unsigned X86::getNFVariantIfClobberRemovable(const MachineInstr &MI,
// optimization for replacing non-NF with NF. This is to keep backward
// compatiblity with old version of linkers without APX relocation type
// support on Linux OS.
- bool IsWithReloc =
- X86EnableAPXForRelocation ? false : isAddMemInstrWithRelocation(MI);
- if (IsWithReloc)
+ if (!X86EnableAPXForRelocation && isAddMemInstrWithRelocation(MI))
return 0;
return X86::getNFVariant(MI.getOpcode());
}
@@ -5320,9 +5318,10 @@ MachineInstr *X86InstrInfo::findDominatingRedundantFlagInstr(
// clobber shadows the producer, so its flags cannot reach CmpInstr.
MachineDominatorTree MDT(MF);
MachineDomTreeNode *Node = MDT.getNode(MultiPredMBB);
+ if (!Node)
+ return nullptr;
MachineInstr *Sub = nullptr;
- for (Node = Node ? Node->getIDom() : nullptr; Node && !Sub;
- Node = Node->getIDom()) {
+ for (Node = Node->getIDom(); Node && !Sub; Node = Node->getIDom()) {
MachineBasicBlock *DomMBB = Node->getBlock();
for (MachineInstr &Inst : reverse(*DomMBB)) {
if (!Inst.modifiesRegister(X86::EFLAGS, TRI))
@@ -5594,7 +5593,14 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
// bb2: ...
// bb3: cmp %x, C ; <-- redundant, reuse EFLAGS from entry
// cmovcc ...
- if (HasNF)
+ //
+ // Only attempt this when no NF conversion has been collected yet. An NF
+ // conversion grows code size, and any already collected lies on the
+ // single-predecessor chain to CmpInstr, i.e. the unconditional path the
+ // compare removal saves, so converting it is pure added cost. Conversions
+ // the dominating walk collects instead lie on the mutually exclusive
+ // diamond arms and are only conditionally executed.
+ if (HasNF && InstsToUpdate.empty())
Sub = findDominatingRedundantFlagInstr(
CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue, MBB, IsSwapped,
ImmDelta, InstsToUpdate);
diff --git a/llvm/test/CodeGen/X86/apx/add.ll b/llvm/test/CodeGen/X86/apx/add.ll
index 8b1e1ceb832a3..d7c5635b617c1 100644
--- a/llvm/test/CodeGen/X86/apx/add.ll
+++ b/llvm/test/CodeGen/X86/apx/add.ll
@@ -1221,24 +1221,26 @@ define i32 @two_address_no_subreg(i32 %arg0, ptr %arg1, i1 %arg2) nounwind {
; NF-NEXT: jne .LBB50_2 # encoding: [0x75,A]
; NF-NEXT: # fixup A - offset: 1, value: .LBB50_2, kind: FK_PCRel_1
; NF-NEXT: # %bb.1: # %bb2
-; NF-NEXT: movl $0, %r12d # encoding: [0x41,0xbc,0x00,0x00,0x00,0x00]
+; NF-NEXT: xorl %r12d, %r12d # encoding: [0x45,0x31,0xe4]
; NF-NEXT: .LBB50_2: # %bb1
-; NF-NEXT: movq (%rsp), %rax # 8-byte Reload
-; NF-NEXT: # encoding: [0x48,0x8b,0x04,0x24]
-; NF-NEXT: {nf} addl %eax, %r13d, %ebp # encoding: [0x62,0xd4,0x54,0x1c,0x01,0xc5]
-; NF-NEXT: {nf} imull %ebx, %ebx, %r13d # encoding: [0x62,0xf4,0x14,0x1c,0xaf,0xdb]
+; NF-NEXT: movl %r13d, %esi # encoding: [0x44,0x89,0xee]
+; NF-NEXT: addl (%rsp), %esi # 4-byte Folded Reload
+; NF-NEXT: # encoding: [0x03,0x34,0x24]
+; NF-NEXT: {nf} imull %ebx, %ebx, %r13d # EVEX TO EVEX Compression encoding: [0x62,0xf4,0x14,0x1c,0xaf,0xdb]
+; NF-NEXT: testb $1, %bpl # encoding: [0x40,0xf6,0xc5,0x01]
; NF-NEXT: jne .LBB50_4 # encoding: [0x75,A]
; NF-NEXT: # fixup A - offset: 1, value: .LBB50_4, kind: FK_PCRel_1
; NF-NEXT: # %bb.3: # %bb4
-; NF-NEXT: xorl %eax, %eax # encoding: [0x31,0xc0]
+; NF-NEXT: xorl %ebp, %ebp # encoding: [0x31,0xed]
; NF-NEXT: movq %r14, %rdi # encoding: [0x4c,0x89,0xf7]
-; NF-NEXT: callq *%rax # encoding: [0xff,0xd0]
-; NF-NEXT: xorl %eax, %eax # encoding: [0x31,0xc0]
+; NF-NEXT: movl %esi, %r14d # encoding: [0x41,0x89,0xf6]
+; NF-NEXT: callq *%rbp # encoding: [0xff,0xd5]
; NF-NEXT: {nf} incl %r12d, %r15d # EVEX TO EVEX Compression encoding: [0x62,0xd4,0x04,0x1c,0xff,0xc4]
; NF-NEXT: xorl %edi, %edi # encoding: [0x31,0xff]
-; NF-NEXT: callq *%rax # encoding: [0xff,0xd0]
+; NF-NEXT: callq *%rbp # encoding: [0xff,0xd5]
+; NF-NEXT: movl %r14d, %esi # encoding: [0x44,0x89,0xf6]
; NF-NEXT: .LBB50_4: # %bb3
-; NF-NEXT: {nf} orl %r13d, %ebp, %esi # EVEX TO EVEX Compression encoding: [0x62,0x74,0x4c,0x1c,0x09,0xed]
+; NF-NEXT: orl %r13d, %esi # EVEX TO LEGACY Compression encoding: [0x44,0x09,0xee]
; NF-NEXT: orl %r15d, %esi # EVEX TO LEGACY Compression encoding: [0x44,0x09,0xfe]
; NF-NEXT: xorl %eax, %eax # encoding: [0x31,0xc0]
; NF-NEXT: movl %ebx, %edi # encoding: [0x89,0xdf]
diff --git a/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir b/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir
index a4af76efe8452..b72389acaa640 100644
--- a/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir
+++ b/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir
@@ -114,6 +114,70 @@ body: |
RET 0, $al
...
+# Profitability: like dom_no_shadow, but bb.3 (the multi-predecessor block)
+# reaches the redundant compare in bb.4 through a single-predecessor chain that
+# itself has an NF-convertible clobber (the IMUL in bb.3). That clobber is on
+# the unconditional path to the compare, so converting it is pure added cost:
+# the reuse is declined and the compare kept (contrast dom_no_shadow, whose only
+# clobbers are on the mutually exclusive diamond arms).
+---
+name: chain_conv_unprofitable
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: chain_conv_unprofitable
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $edi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: JCC_1 %bb.2, 12, implicit $eflags
+ ; CHECK-NEXT: JMP_1 %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[IMUL32rr:%[0-9]+]]:gr32 = nuw nsw IMUL32rr [[COPY]], [[COPY1]], implicit-def dead $eflags
+ ; CHECK-NEXT: JMP_1 %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[IMUL32rr1:%[0-9]+]]:gr32 = nuw nsw IMUL32rr [[COPY]], [[COPY1]], implicit-def dead $eflags
+ ; CHECK-NEXT: JMP_1 %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: $al = SETCCr 15, implicit $eflags
+ ; CHECK-NEXT: RET 0, $al
+ bb.0:
+ liveins: $edi, $esi
+ %0:gr32 = COPY $edi
+ %1:gr32 = COPY $esi
+ CMP32ri %0, 2, implicit-def $eflags
+ JCC_1 %bb.2, 12, implicit $eflags
+ JMP_1 %bb.1
+
+ bb.1:
+ %2:gr32 = nuw nsw IMUL32rr %0, %1, implicit-def dead $eflags
+ JMP_1 %bb.3
+
+ bb.2:
+
+ bb.3:
+ %3:gr32 = nuw nsw IMUL32rr %0, %1, implicit-def dead $eflags
+ JMP_1 %bb.4
+
+ bb.4:
+ CMP32ri %0, 2, implicit-def $eflags
+ $al = SETCCr 15, implicit $eflags
+ RET 0, $al
+...
+
# Loop: the producer in bb.0 dominates the multi-predecessor block bb.1 (preds
# bb.0 and the bb.3 back-edge). The redundant compare in bb.1 is removed and the
# NF-convertible IMUL on the bb.2 path is rewritten. This exercises the
>From 93a8e8b23c6ddb1145e84a68fae46790926653e7 Mon Sep 17 00:00:00 2001
From: Feng Zou <feng.zou at intel.com>
Date: Sat, 11 Jul 2026 22:31:16 +0800
Subject: [PATCH 4/4] [X86] Avoid per-compare dominator-tree build in
optimizeCompareInstr
findDominatingRedundantFlagInstr built a whole-function MachineDominatorTree on
every call, i.e. once per redundant cross-block compare -- O(compares *
function size), quadratic. A microbenchmark of 1200 diamonds in one function
showed the +nf path adding ~0.55s over baseline.
Replace it with a single backward CFG walk from the multi-predecessor block
that finds the dominating flag producer and verifies dominance (bail if a
backward path reaches a function-entry block without the producer, or a second
producer block appears). Work is now bounded by the region reaching the compare
rather than the whole function; the 1200-diamond case drops to the -nf baseline
with byte-identical output. The now-unused MachineDominators.h include is
dropped.
Add MIR tests producer_in_nondominating_pred and producer_in_two_blocks for the
two dominance-check bail paths.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply at anthropic.com>
---
llvm/lib/Target/X86/X86InstrInfo.cpp | 117 ++++++++---------
.../X86/apx/optimize-compare-multipred.mir | 118 +++++++++++++++++-
2 files changed, 161 insertions(+), 74 deletions(-)
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index a598437789f7f..225de900c1454 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -23,7 +23,6 @@
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/LiveVariables.h"
#include "llvm/CodeGen/MachineConstantPool.h"
-#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -5304,40 +5303,65 @@ MachineInstr *X86InstrInfo::findDominatingRedundantFlagInstr(
int64_t &ImmDelta,
SmallVectorImpl<std::pair<MachineInstr *, unsigned>> &InstsToUpdate) const {
assert(Subtarget.hasNF() && "NF feature required");
- MachineBasicBlock &CmpMBB = *CmpInstr.getParent();
- MachineFunction &MF = *CmpMBB.getParent();
const TargetRegisterInfo *TRI = &getRegisterInfo();
- // The caller already scanned MultiPredMBB (and any single-predecessor blocks
- // between it and CmpMBB) without finding the producer, so the producer must
- // live in a block that strictly dominates MultiPredMBB. Walk up the
- // immediate-dominator chain. In each dominating block scan backward: the
- // redundant producer ends the search; an NF-convertible clobber is stepped
- // over (it will be collected by the path walk below) so the producer can
- // still be found earlier in the same block or further up; any other EFLAGS
- // clobber shadows the producer, so its flags cannot reach CmpInstr.
- MachineDominatorTree MDT(MF);
- MachineDomTreeNode *Node = MDT.getNode(MultiPredMBB);
- if (!Node)
- return nullptr;
+ // The caller already scanned MultiPredMBB without finding the producer, so it
+ // must live in a block that strictly dominates MultiPredMBB. Walk
+ // predecessors backward to find it and prove dominance, avoiding a
+ // whole-function MachineDominatorTree that would be rebuilt in O(function
+ // size) per compare.
+ //
+ // The producer's block dominates MultiPredMBB iff every backward path funnels
+ // through it before a function-entry block, so expand predecessors but stop
+ // at a block holding the producer. Bail if a predecessor-less block is
+ // reached without the producer (a path bypasses it) or the producer is found
+ // in two blocks (neither dominates alone). Within a block, scan backward,
+ // collecting the NF-convertible EFLAGS clobbers above the producer and
+ // bailing on any other clobber (it would shadow the producer's flags from
+ // CmpInstr).
+ //
+ // Clobbers are staged in Pending and committed only on success. Visited
+ // (seeded with MultiPredMBB) stops the walk from revisiting a block or
+ // re-entering the single-predecessor chain, so none is collected twice.
MachineInstr *Sub = nullptr;
- for (Node = Node->getIDom(); Node && !Sub; Node = Node->getIDom()) {
- MachineBasicBlock *DomMBB = Node->getBlock();
- for (MachineInstr &Inst : reverse(*DomMBB)) {
+ MachineBasicBlock *SubMBB = nullptr;
+ SmallVector<std::pair<MachineInstr *, unsigned>, 4> Pending;
+ SmallPtrSet<MachineBasicBlock *, 8> Visited;
+ SmallVector<MachineBasicBlock *, 8> Worklist;
+ Visited.insert(MultiPredMBB);
+ for (MachineBasicBlock *Pred : MultiPredMBB->predecessors())
+ if (Visited.insert(Pred).second)
+ Worklist.push_back(Pred);
+ while (!Worklist.empty()) {
+ MachineBasicBlock *MBB = Worklist.pop_back_val();
+ MachineInstr *Producer = nullptr;
+ for (MachineInstr &Inst : reverse(*MBB)) {
if (!Inst.modifiesRegister(X86::EFLAGS, TRI))
continue;
if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue,
Inst, &IsSwapped, &ImmDelta)) {
- // Found the producer.
- Sub = &Inst;
+ Producer = &Inst;
break;
}
- if (!X86::getNFVariantIfClobberRemovable(Inst, TRI))
- // A non-convertible clobber shadows any producer further up.
+ unsigned NewOpc = X86::getNFVariantIfClobberRemovable(Inst, TRI);
+ if (!NewOpc)
return nullptr;
- // NF-convertible clobber: keep scanning earlier in this block.
+ Pending.push_back(std::make_pair(&Inst, NewOpc));
}
- // No producer in this block: keep walking up the dominator chain.
+ if (Producer) {
+ // A producer in a second block means neither dominates alone.
+ if (Sub && SubMBB != MBB)
+ return nullptr;
+ Sub = Producer;
+ SubMBB = MBB;
+ continue;
+ }
+ // Entry reached without the producer: some path bypasses it.
+ if (MBB->pred_empty())
+ return nullptr;
+ for (MachineBasicBlock *Pred : MBB->predecessors())
+ if (Visited.insert(Pred).second)
+ Worklist.push_back(Pred);
}
if (!Sub)
return nullptr;
@@ -5350,51 +5374,8 @@ MachineInstr *X86InstrInfo::findDominatingRedundantFlagInstr(
// to producers that yield identical flags.
if (IsSwapped || ImmDelta != 0)
return nullptr;
- MachineBasicBlock *SubMBB = Sub->getParent();
-
- // Verify that on every path from the producer to CmpInstr all EFLAGS clobbers
- // are NF-convertible, collecting them. The caller already handled CmpMBB, the
- // single-predecessor chain, and MultiPredMBB itself, so start from
- // MultiPredMBB's predecessors and walk the CFG backward up to (and including
- // the post-producer range of) SubMBB. Because SubMBB dominates MultiPredMBB,
- // every block that can reach MultiPredMBB is dominated by SubMBB, so the walk
- // reaches SubMBB on every path and never escapes above it.
- //
- // The walk never revisits a block (Visited) and never reaches a block the
- // caller already scanned: MultiPredMBB is seeded into Visited, the
- // single-predecessor chain blocks (CmpMBB..MultiPredMBB) each have exactly
- // one predecessor by construction (the caller only advanced through such
- // blocks), so none can be re-entered from the dominated region, and CmpMBB
- // lies in the successor direction. No instruction is therefore collected
- // twice.
- SmallPtrSet<MachineBasicBlock *, 8> Visited;
- SmallVector<MachineBasicBlock *, 8> Worklist;
- Visited.insert(MultiPredMBB);
- for (MachineBasicBlock *Pred : MultiPredMBB->predecessors())
- if (Visited.insert(Pred).second)
- Worklist.push_back(Pred);
- while (!Worklist.empty()) {
- MachineBasicBlock *MBB = Worklist.pop_back_val();
- // For SubMBB only the range after the producer is relevant.
- MachineBasicBlock::iterator I =
- (MBB == SubMBB) ? std::next(MachineBasicBlock::iterator(Sub))
- : MBB->begin();
- for (MachineBasicBlock::iterator E = MBB->end(); I != E; ++I) {
- MachineInstr &MI = *I;
- if (!MI.modifiesRegister(X86::EFLAGS, TRI))
- continue;
- unsigned NewOpc = X86::getNFVariantIfClobberRemovable(MI, TRI);
- if (!NewOpc)
- return nullptr;
- InstsToUpdate.push_back(std::make_pair(&MI, NewOpc));
- }
- if (MBB == SubMBB)
- continue;
- for (MachineBasicBlock *Pred : MBB->predecessors())
- if (Visited.insert(Pred).second)
- Worklist.push_back(Pred);
- }
+ InstsToUpdate.append(Pending.begin(), Pending.end());
return Sub;
}
diff --git a/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir b/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir
index b72389acaa640..39d469c0fcb59 100644
--- a/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir
+++ b/llvm/test/CodeGen/X86/apx/optimize-compare-multipred.mir
@@ -8,8 +8,6 @@
# A non-NF-convertible EFLAGS clobber in the dominating block that defines the
# flags shadows the producer, so the redundant compare in bb.3 must be kept.
-# (The clobber is rejected by both the idom-chain scan and the subsequent
-# path walk; either alone is sufficient to bail.)
---
name: dom_shadow_clobber
tracksRegLiveness: true
@@ -180,10 +178,10 @@ body: |
# Loop: the producer in bb.0 dominates the multi-predecessor block bb.1 (preds
# bb.0 and the bb.3 back-edge). The redundant compare in bb.1 is removed and the
-# NF-convertible IMUL on the bb.2 path is rewritten. This exercises the
-# verification and live-in walks over a cyclic CFG with a back-edge (the Visited
-# sets keep both walks terminating), which the acyclic diamonds in
-# optimize-compare-multipred.ll do not cover.
+# NF-convertible IMUL on the bb.2 path is rewritten. This exercises the backward
+# walk over a cyclic CFG with a back-edge (the Visited set keeps it
+# terminating), which the acyclic diamonds in optimize-compare-multipred.ll do
+# not cover.
---
name: loop_backedge
tracksRegLiveness: true
@@ -245,3 +243,111 @@ body: |
$al = SETCCr 15, implicit $eflags
RET 0, $al
...
+
+# The producer lives in bb.2, an immediate predecessor of the multi-predecessor
+# block bb.3, but bb.3's other predecessor bb.1 reaches it without passing
+# through bb.2, so bb.2 does not dominate bb.3. The backward walk reaches the
+# entry block bb.0 (predecessor-less) without finding the producer on that path
+# and bails, so the redundant compare in bb.3 is kept.
+---
+name: producer_in_nondominating_pred
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: producer_in_nondominating_pred
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $edi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: JCC_1 %bb.2, 4, implicit undef $eflags
+ ; CHECK-NEXT: JMP_1 %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: JMP_1 %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: JMP_1 %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: $al = SETCCr 15, implicit $eflags
+ ; CHECK-NEXT: RET 0, $al
+ bb.0:
+ liveins: $edi, $esi
+ %0:gr32 = COPY $edi
+ %1:gr32 = COPY $esi
+ JCC_1 %bb.2, 4, implicit undef $eflags
+ JMP_1 %bb.1
+
+ bb.1:
+ JMP_1 %bb.3
+
+ bb.2:
+ CMP32ri %0, 2, implicit-def $eflags
+ JMP_1 %bb.3
+
+ bb.3:
+ CMP32ri %0, 2, implicit-def $eflags
+ $al = SETCCr 15, implicit $eflags
+ RET 0, $al
+...
+
+# The producer is duplicated on both diamond arms bb.1 and bb.2, so neither
+# block alone dominates the multi-predecessor block bb.3. The backward walk
+# finds a producer in two distinct blocks and bails, keeping the compare.
+---
+name: producer_in_two_blocks
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: producer_in_two_blocks
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.2(0x40000000), %bb.1(0x40000000)
+ ; CHECK-NEXT: liveins: $edi, $esi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $edi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
+ ; CHECK-NEXT: JCC_1 %bb.2, 4, implicit undef $eflags
+ ; CHECK-NEXT: JMP_1 %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: JMP_1 %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: JMP_1 %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: CMP32ri [[COPY]], 2, implicit-def $eflags
+ ; CHECK-NEXT: $al = SETCCr 15, implicit $eflags
+ ; CHECK-NEXT: RET 0, $al
+ bb.0:
+ liveins: $edi, $esi
+ %0:gr32 = COPY $edi
+ %1:gr32 = COPY $esi
+ JCC_1 %bb.2, 4, implicit undef $eflags
+ JMP_1 %bb.1
+
+ bb.1:
+ CMP32ri %0, 2, implicit-def $eflags
+ JMP_1 %bb.3
+
+ bb.2:
+ CMP32ri %0, 2, implicit-def $eflags
+ JMP_1 %bb.3
+
+ bb.3:
+ CMP32ri %0, 2, implicit-def $eflags
+ $al = SETCCr 15, implicit $eflags
+ RET 0, $al
+...
More information about the llvm-commits
mailing list