[llvm] [AArch64] Transform lsr + cbz/cbnz into cmp + bcc in loops (PR #228005)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 1 07:54:13 PDT 2026
https://github.com/KRM7 updated https://github.com/llvm/llvm-project/pull/228005
>From 861f3c27f15b0aba858b7d0b8cf2519270988098 Mon Sep 17 00:00:00 2001
From: Krisztian Rugasi <krisztian.rugasi at arm.com>
Date: Tue, 29 Sep 2026 11:57:34 +0000
Subject: [PATCH 1/2] [AArch64] Add tests for (lsr + cbz/cbnz) to (cmp + bcc)
transforms
---
llvm/test/CodeGen/AArch64/cond-br-tuning.ll | 107 ++++++-
llvm/test/CodeGen/AArch64/cond-br-tuning.mir | 306 +++++++++++++++++++
2 files changed, 412 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AArch64/cond-br-tuning.mir
diff --git a/llvm/test/CodeGen/AArch64/cond-br-tuning.ll b/llvm/test/CodeGen/AArch64/cond-br-tuning.ll
index 87a315a49f3e0..3c3f4c8040c6b 100644
--- a/llvm/test/CodeGen/AArch64/cond-br-tuning.ll
+++ b/llvm/test/CodeGen/AArch64/cond-br-tuning.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -debugify-and-strip-all-safe < %s -O3 -mtriple=aarch64 -verify-machineinstrs | FileCheck %s
+; RUN: llc -debugify-and-strip-all-safe < %s -O3 -mtriple=aarch64 -mattr=cmp-bcc-fusion -verify-machineinstrs | FileCheck %s
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64"
@@ -215,3 +215,108 @@ if.then:
if.end:
ret void
}
+
+define void @test_lsr_cbnz(ptr %p0) {
+; CHECK-LABEL: test_lsr_cbnz:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, x0
+; CHECK-NEXT: .LBB10_1: // %loop.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldr x9, [x8], #32
+; CHECK-NEXT: lsr x10, x9, #32
+; CHECK-NEXT: cbnz x10, .LBB10_1
+; CHECK-NEXT: // %bb.2: // %loop.exit
+; CHECK-NEXT: str x9, [x0]
+; CHECK-NEXT: ret
+entry:
+ br label %loop.body
+
+loop.body:
+ %p = phi ptr [ %p0, %entry ], [ %pn, %loop.body ]
+ %val = load i64, ptr %p, align 8
+ %pn = getelementptr i8, ptr %p, i64 32
+ %cmp = icmp ugt i64 %val, 4294967295
+ br i1 %cmp, label %loop.body, label %loop.exit
+
+loop.exit:
+ store i64 %val, ptr %p0, align 8
+ ret void
+}
+
+define void @test_lsr_cbz(ptr %p0) {
+; CHECK-LABEL: test_lsr_cbz:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, x0
+; CHECK-NEXT: .LBB11_1: // %loop.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldr x9, [x8], #32
+; CHECK-NEXT: lsr x10, x9, #32
+; CHECK-NEXT: cbz x10, .LBB11_1
+; CHECK-NEXT: // %bb.2: // %loop.exit
+; CHECK-NEXT: str x9, [x0]
+; CHECK-NEXT: ret
+entry:
+ br label %loop.body
+
+loop.body:
+ %p = phi ptr [ %p0, %entry ], [ %pn, %loop.body ]
+ %val = load i64, ptr %p, align 8
+ %pn = getelementptr i8, ptr %p, i64 32
+ %cmp = icmp ule i64 %val, 4294967295
+ br i1 %cmp, label %loop.body, label %loop.exit
+
+loop.exit:
+ store i64 %val, ptr %p0, align 8
+ ret void
+}
+
+define void @test_lshr_icmp0(ptr %p0) {
+; CHECK-LABEL: test_lshr_icmp0:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov x8, x0
+; CHECK-NEXT: .LBB12_1: // %loop.body
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ldr x9, [x8], #32
+; CHECK-NEXT: lsr x10, x9, #36
+; CHECK-NEXT: cbnz x10, .LBB12_1
+; CHECK-NEXT: // %bb.2: // %loop.exit
+; CHECK-NEXT: str x9, [x0]
+; CHECK-NEXT: ret
+entry:
+ br label %loop.body
+
+loop.body:
+ %p = phi ptr [ %p0, %entry ], [ %pn, %loop.body ]
+ %val = load i64, ptr %p, align 8
+ %pn = getelementptr i8, ptr %p, i64 32
+ %shl = lshr i64 %val, 36
+ %cmp = icmp ne i64 %shl, 0
+ br i1 %cmp, label %loop.body, label %loop.exit
+
+loop.exit:
+ store i64 %val, ptr %p0, align 8
+ ret void
+}
+
+define void @test_cant_hoist_cmp_imm(ptr %p0, ptr %p1) {
+; CHECK-LABEL: test_cant_hoist_cmp_imm:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldr x8, [x0]
+; CHECK-NEXT: lsr x9, x8, #32
+; CHECK-NEXT: cbnz x9, .LBB13_2
+; CHECK-NEXT: // %bb.1: // %exit1
+; CHECK-NEXT: str x8, [x1]
+; CHECK-NEXT: .LBB13_2: // %exit2
+; CHECK-NEXT: ret
+entry:
+ %val = load i64, ptr %p0, align 8
+ %cmp = icmp ule i64 %val, 4294967295
+ br i1 %cmp, label %exit1, label %exit2
+
+exit1:
+ store i64 %val, ptr %p1, align 8
+ br label %exit2
+
+exit2:
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/cond-br-tuning.mir b/llvm/test/CodeGen/AArch64/cond-br-tuning.mir
new file mode 100644
index 0000000000000..691f88d049b38
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cond-br-tuning.mir
@@ -0,0 +1,306 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -mattr=cmp-bcc-fusion -run-pass=aarch64-cond-br-tuning -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: test_ubfmx32_cbnzx
+body: |
+ ; CHECK-LABEL: name: test_ubfmx32_cbnzx
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 32, 63
+ ; CHECK-NEXT: CBNZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ %0:gpr64 = COPY $x0
+
+ bb.1:
+ %1:gpr64 = UBFMXri %0, 32, 63
+ CBNZX %1, %bb.1
+
+ bb.2:
+ RET_ReallyLR
+...
+
+---
+name: test_ubfmx47_cbnzx
+body: |
+ ; CHECK-LABEL: name: test_ubfmx47_cbnzx
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 47, 63
+ ; CHECK-NEXT: CBNZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ %0:gpr64 = COPY $x0
+
+ bb.1:
+ %1:gpr64 = UBFMXri %0, 47, 63
+ CBNZX %1, %bb.1
+
+ bb.2:
+ RET_ReallyLR
+...
+
+---
+name: test_ubfmw16_cbzw
+body: |
+ ; CHECK-LABEL: name: test_ubfmw16_cbzw
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY]], 16, 31
+ ; CHECK-NEXT: CBZW [[UBFMWri]], %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ %0:gpr32 = COPY $w0
+
+ bb.1:
+ %1:gpr32 = UBFMWri %0, 16, 31
+ CBZW %1, %bb.1
+
+ bb.2:
+ RET_ReallyLR
+...
+
+---
+name: test_ubfmx_cbnzw_subreg_only
+body: |
+ ; CHECK-LABEL: name: test_ubfmx_cbnzw_subreg_only
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 32, 63
+ ; CHECK-NEXT: CBNZW [[UBFMXri]].sub_32, %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ %0:gpr64 = COPY $x0
+
+ bb.1:
+ %1:gpr64 = UBFMXri %0, 32, 63
+ CBNZW %1.sub_32, %bb.1
+
+ bb.2:
+ RET_ReallyLR
+...
+
+---
+name: test_multiuse_ubfm
+body: |
+ ; CHECK-LABEL: name: test_multiuse_ubfm
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 32, 63
+ ; CHECK-NEXT: CBNZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: FAKE_USE [[UBFMXri]]
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ %0:gpr64 = COPY $x0
+
+ bb.1:
+ %1:gpr64 = UBFMXri %0, 32, 63
+ CBNZX %1, %bb.1
+
+ bb.2:
+ FAKE_USE %1
+ RET_ReallyLR
+...
+
+---
+name: test_not_lsr_ubfm
+body: |
+ ; CHECK-LABEL: name: test_not_lsr_ubfm
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 32, 62
+ ; CHECK-NEXT: CBNZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ %0:gpr64 = COPY $x0
+
+ bb.1:
+ %1:gpr64 = UBFMXri %0, 32, 62
+ CBNZX %1, %bb.1
+
+ bb.2:
+ RET_ReallyLR
+...
+
+---
+name: test_nzcv_liveout
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test_nzcv_liveout
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: HINT 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 32, 63
+ ; CHECK-NEXT: CBNZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: liveins: $nzcv
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: RET_ReallyLR implicit $nzcv
+ bb.0:
+ liveins: $x0
+ %0:gpr64 = COPY $x0
+
+ bb.1:
+ HINT 0, implicit-def $nzcv
+ %1:gpr64 = UBFMXri %0, 32, 63
+ CBNZX %1, %bb.1
+
+ bb.2:
+ liveins: $nzcv
+ RET_ReallyLR implicit $nzcv
+...
+
+---
+name: test_nzcv_live_at_cbnz
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test_nzcv_live_at_cbnz
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: HINT 0, implicit-def $nzcv
+ ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 32, 63
+ ; CHECK-NEXT: CBNZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: Bcc 0, %bb.1, implicit $nzcv
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ liveins: $x0
+ %0:gpr64 = COPY $x0
+
+ bb.1:
+ HINT 0, implicit-def $nzcv
+ %1:gpr64 = UBFMXri %0, 32, 63
+ CBNZX %1, %bb.1
+ Bcc 0, %bb.1, implicit $nzcv
+
+ bb.2:
+ RET_ReallyLR
+...
+
+---
+name: test_nzcv_clobber
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: test_nzcv_clobber
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 32, 63
+ ; CHECK-NEXT: CBNZX [[COPY]], %bb.1, implicit-def $nzcv
+ ; CHECK-NEXT: CBNZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ liveins: $x0
+ %0:gpr64 = COPY $x0
+
+ bb.1:
+ %1:gpr64 = UBFMXri %0, 32, 63
+ CBNZX %0, %bb.1, implicit-def $nzcv
+ CBNZX %1, %bb.1
+
+ bb.2:
+ RET_ReallyLR
+...
+
+---
+name: test_multiple_terminators
+body: |
+ ; CHECK-LABEL: name: test_multiple_terminators
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 32, 63
+ ; CHECK-NEXT: CBNZX [[COPY]], %bb.1
+ ; CHECK-NEXT: CBZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: RET_ReallyLR
+ bb.0:
+ %0:gpr64 = COPY $x0
+
+ bb.1:
+ %1:gpr64 = UBFMXri %0, 32, 63
+ CBNZX %0, %bb.1
+ CBZX %1, %bb.1
+
+ bb.2:
+ RET_ReallyLR
+...
>From 2b79340fb32e283554a09e6ec3ec27f65b0de43a Mon Sep 17 00:00:00 2001
From: Krisztian Rugasi <krisztian.rugasi at arm.com>
Date: Tue, 29 Sep 2026 11:58:02 +0000
Subject: [PATCH 2/2] [AArch64] Transform lsr + cbz/cbnz into cmp + bcc in
loops
Transform an lsr + cbz/cbnz instruction pair into a cmp + bcc pair
when beneficial as:
lsr x0, x0, imm
cbz x0, .LBB0
--------------->
mov x1, 2^imm
cmp x0, x1
bcc .LBB0
This results in an additional instruction used for materializing
the immediate for the cmp instruction, but it is beneficial when
this instruction can be hoisted out of the block and the cmp and
bcc instructions are fused.
---
.../Target/AArch64/AArch64CondBrTuning.cpp | 147 +++++++++++++++++-
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 2 +-
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 2 +-
.../AArch64/cond-br-tuning-instr-ref.mir | 7 -
llvm/test/CodeGen/AArch64/cond-br-tuning.ll | 33 ++--
llvm/test/CodeGen/AArch64/cond-br-tuning.mir | 20 ++-
6 files changed, 175 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64CondBrTuning.cpp b/llvm/lib/Target/AArch64/AArch64CondBrTuning.cpp
index 4d34487528910..499756cbbf9d6 100644
--- a/llvm/lib/Target/AArch64/AArch64CondBrTuning.cpp
+++ b/llvm/lib/Target/AArch64/AArch64CondBrTuning.cpp
@@ -23,13 +23,19 @@
/// 3) sub w8, w0, w1 -> subs w8, w0, w1 ; w8 has multiple uses.
/// tbz w8, #31, .LBB6_2 -> b.pl .LBB6_2
///
+/// 4) lsr x8, x0, 32 -> mov w9, #4294967296 ; will be hoisted
+/// cbz x8, .LBB0_0 -> cmp x8, x9 ; fused with bcc
+/// -> b.lo .LBB0_0
+///
//===----------------------------------------------------------------------===//
#include "AArch64.h"
+#include "AArch64InstrInfo.h"
#include "AArch64Subtarget.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
@@ -45,8 +51,10 @@ using namespace llvm;
namespace {
class AArch64CondBrTuning : public MachineFunctionPass {
+ const AArch64Subtarget *STI;
const AArch64InstrInfo *TII;
const TargetRegisterInfo *TRI;
+ const MachineLoopInfo *MLI;
MachineRegisterInfo *MRI;
@@ -63,6 +71,7 @@ class AArch64CondBrTuning : public MachineFunctionPass {
bool Is64Bit);
MachineInstr *convertToCondBr(MachineInstr &MI);
bool tryToTuneBranch(MachineInstr &MI, MachineInstr &DefMI);
+ bool tryTransformShiftBranch(MachineBasicBlock &MBB, MachineInstr &MI) const;
};
} // end anonymous namespace
@@ -73,9 +82,17 @@ INITIALIZE_PASS(AArch64CondBrTuning, "aarch64-cond-br-tuning",
void AArch64CondBrTuning::getAnalysisUsage(AnalysisUsage &AU) const {
AU.setPreservesCFG();
+ AU.addRequired<MachineLoopInfoWrapperPass>();
+ AU.addPreserved<MachineLoopInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
+static bool isNZCVLiveout(const MachineBasicBlock &MBB) {
+ return any_of(MBB.successors(), [](const MachineBasicBlock *SuccMBB) {
+ return SuccMBB->isLiveIn(AArch64::NZCV);
+ });
+}
+
MachineInstr *AArch64CondBrTuning::getOperandDef(const MachineOperand &MO) {
if (!MO.getReg().isVirtual())
return nullptr;
@@ -202,7 +219,7 @@ bool AArch64CondBrTuning::tryToTuneBranch(MachineInstr &MI,
// There must not be any instruction between DefMI and MI that clobbers or
// reads NZCV.
- if (isNZCVTouchedInInstructionRange(DefMI, MI, TRI))
+ if (isNZCVTouchedInInstructionRange(*DefMI.getNextNode(), MI, TRI))
return false;
NewCmp = tryConvertToFlagSetting(DefMI, IsFlagSetting, /*Is64Bit=*/false);
@@ -261,7 +278,7 @@ bool AArch64CondBrTuning::tryToTuneBranch(MachineInstr &MI,
return false;
// There must not be any instruction between DefMI and MI that clobbers or
// reads NZCV.
- if (isNZCVTouchedInInstructionRange(DefMI, MI, TRI))
+ if (isNZCVTouchedInInstructionRange(*DefMI.getNextNode(), MI, TRI))
return false;
NewCmp = tryConvertToFlagSetting(DefMI, IsFlagSetting, /*Is64Bit=*/true);
@@ -296,6 +313,105 @@ bool AArch64CondBrTuning::tryToTuneBranch(MachineInstr &MI,
return true;
}
+/// Try to transform lsr + cbz/cbnz instruction pairs into a cmp + bcc pair.
+/// This will require an additional instruction to materialize the cmp
+/// immediate, but it saves an instruction when the imm mat instruction
+/// can be hoisted out of the block and the cmp + bcc instructions are fused.
+bool AArch64CondBrTuning::tryTransformShiftBranch(MachineBasicBlock &MBB,
+ MachineInstr &MI) const {
+ switch (MI.getOpcode()) {
+ default:
+ break;
+ case AArch64::CBNZW:
+ case AArch64::CBZW: {
+ const Register CondReg = MI.getOperand(0).getReg();
+ if (!CondReg.isVirtual() || !MRI->hasOneNonDBGUse(CondReg))
+ return false;
+
+ MachineInstr &CondDef = *MRI->getUniqueVRegDef(CondReg);
+ if (CondDef.getOpcode() != AArch64::UBFMWri ||
+ CondDef.getOperand(3).getImm() != 31)
+ return false;
+
+ // NZCV can't be live after the last non-terminator in the block for the
+ // transformation to work. At this point we know it's not live after MI,
+ // so we only have to check the rest of the range here.
+ if (isNZCVTouchedInInstructionRange(*MBB.getFirstTerminator(), MI, TRI))
+ return false;
+
+ const uint64_t ShiftAmt = CondDef.getOperand(2).getImm();
+ const uint64_t CmpImm = 1ULL << ShiftAmt;
+
+ const AArch64CC::CondCode CC =
+ (MI.getOpcode() == AArch64::CBZW) ? AArch64CC::LO : AArch64CC::HS;
+
+ const Register ImmReg = MRI->createVirtualRegister(&AArch64::GPR32RegClass);
+
+ BuildMI(MBB, MBB.getFirstTerminator(), CondDef.getDebugLoc(),
+ TII->get(AArch64::MOVi32imm))
+ .addDef(ImmReg)
+ .addImm(CmpImm);
+
+ BuildMI(MBB, MBB.getFirstTerminator(), CondDef.getDebugLoc(),
+ TII->get(AArch64::SUBSWrr))
+ .addDef(AArch64::WZR)
+ .addReg(CondDef.getOperand(1).getReg())
+ .addReg(ImmReg);
+
+ BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AArch64::Bcc))
+ .addImm(CC)
+ .addMBB(MI.getOperand(1).getMBB());
+
+ CondDef.eraseFromParent();
+ MI.eraseFromParent();
+ return true;
+ }
+ case AArch64::CBNZX:
+ case AArch64::CBZX: {
+ const Register CondReg = MI.getOperand(0).getReg();
+ if (!CondReg.isVirtual() || !MRI->hasOneNonDBGUse(CondReg))
+ return false;
+
+ MachineInstr &CondDef = *MRI->getUniqueVRegDef(CondReg);
+ if (CondDef.getOpcode() != AArch64::UBFMXri ||
+ CondDef.getOperand(3).getImm() != 63)
+ return false;
+
+ if (isNZCVTouchedInInstructionRange(*MBB.getFirstTerminator(), MI, TRI))
+ return false;
+
+ const uint64_t ShiftAmt = CondDef.getOperand(2).getImm();
+ const uint64_t CmpImm = 1ULL << ShiftAmt;
+
+ const AArch64CC::CondCode CC =
+ (MI.getOpcode() == AArch64::CBZX) ? AArch64CC::LO : AArch64CC::HS;
+
+ const Register ImmReg = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
+
+ BuildMI(MBB, MBB.getFirstTerminator(), CondDef.getDebugLoc(),
+ TII->get(AArch64::MOVi64imm))
+ .addDef(ImmReg)
+ .addImm(CmpImm);
+
+ BuildMI(MBB, MBB.getFirstTerminator(), CondDef.getDebugLoc(),
+ TII->get(AArch64::SUBSXrr))
+ .addDef(AArch64::XZR)
+ .addReg(CondDef.getOperand(1).getReg())
+ .addReg(ImmReg);
+
+ BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(AArch64::Bcc))
+ .addImm(CC)
+ .addMBB(MI.getOperand(1).getMBB());
+
+ CondDef.eraseFromParent();
+ MI.eraseFromParent();
+ return true;
+ }
+ }
+
+ return false;
+}
+
bool AArch64CondBrTuning::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
@@ -304,8 +420,10 @@ bool AArch64CondBrTuning::runOnMachineFunction(MachineFunction &MF) {
dbgs() << "********** AArch64 Conditional Branch Tuning **********\n"
<< "********** Function: " << MF.getName() << '\n');
- TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo());
- TRI = MF.getSubtarget().getRegisterInfo();
+ STI = &MF.getSubtarget<AArch64Subtarget>();
+ TII = static_cast<const AArch64InstrInfo *>(STI->getInstrInfo());
+ TRI = STI->getRegisterInfo();
+ MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
MRI = &MF.getRegInfo();
bool Changed = false;
@@ -335,6 +453,27 @@ bool AArch64CondBrTuning::runOnMachineFunction(MachineFunction &MF) {
}
}
}
+
+ if (!STI->hasCmpBccFusion())
+ return Changed;
+
+ for (const MachineLoop *ML : *MLI) {
+ for (MachineBasicBlock *MBB : ML->blocks()) {
+ if (isNZCVLiveout(*MBB))
+ continue;
+
+ for (MachineInstr &MI : reverse(MBB->terminators())) {
+ if (MI.readsRegister(AArch64::NZCV, TRI))
+ break;
+
+ if (tryTransformShiftBranch(*MBB, MI)) {
+ Changed = true;
+ break;
+ }
+ }
+ }
+ }
+
return Changed;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 885b27677f786..fe14c286d7262 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -6804,7 +6804,7 @@ void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
const MachineInstr &UseMI,
const TargetRegisterInfo *TRI) {
- return any_of(instructionsWithoutDebug(std::next(DefMI.getIterator()),
+ return any_of(instructionsWithoutDebug(DefMI.getIterator(),
UseMI.getIterator()),
[TRI](const MachineInstr &I) {
return I.modifiesRegister(AArch64::NZCV, TRI) ||
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 6a8346395eea1..e83a57d4fdf4f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -678,7 +678,7 @@ examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
const TargetRegisterInfo &TRI,
SmallVectorImpl<MachineInstr *> *CCUseInstrs = nullptr);
-/// Return true if there is an instruction /after/ \p DefMI and before \p UseMI
+/// Return true if there is an instruction in the range [ \p DefMI, \p UseMI )
/// which either reads or clobbers NZCV.
bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
const MachineInstr &UseMI,
diff --git a/llvm/test/CodeGen/AArch64/cond-br-tuning-instr-ref.mir b/llvm/test/CodeGen/AArch64/cond-br-tuning-instr-ref.mir
index 589102ed63878..a3715122f9faa 100644
--- a/llvm/test/CodeGen/AArch64/cond-br-tuning-instr-ref.mir
+++ b/llvm/test/CodeGen/AArch64/cond-br-tuning-instr-ref.mir
@@ -8,13 +8,6 @@
# CHECK-NEXT: DBG_INSTR_REF !{{[0-9]+}}, !DIExpression(DW_OP_LLVM_arg, 0), dbg-instr-ref(1, 0)
--- |
- define ptr @_ZNK4llvm5Error6getPtrEv(ptr %this) local_unnamed_addr #0 {
- entry:
- %0 = ptrtoint ptr %this to i64
- %and = and i64 %0, -2
- %1 = inttoptr i64 %and to ptr
- ret ptr %1
- }
define noalias noundef ptr @_ZN4llvmlsERNS_11raw_ostreamERKNS_5ErrorE(ptr %E) local_unnamed_addr #1 !dbg !4 {
entry:
%0 = ptrtoint ptr %E to i64, !dbg !13
diff --git a/llvm/test/CodeGen/AArch64/cond-br-tuning.ll b/llvm/test/CodeGen/AArch64/cond-br-tuning.ll
index 3c3f4c8040c6b..717325eaa27ae 100644
--- a/llvm/test/CodeGen/AArch64/cond-br-tuning.ll
+++ b/llvm/test/CodeGen/AArch64/cond-br-tuning.ll
@@ -219,14 +219,15 @@ if.end:
define void @test_lsr_cbnz(ptr %p0) {
; CHECK-LABEL: test_lsr_cbnz:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, x0
+; CHECK-NEXT: mov x8, #4294967296 // =0x100000000
+; CHECK-NEXT: mov x9, x0
; CHECK-NEXT: .LBB10_1: // %loop.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldr x9, [x8], #32
-; CHECK-NEXT: lsr x10, x9, #32
-; CHECK-NEXT: cbnz x10, .LBB10_1
+; CHECK-NEXT: ldr x10, [x9], #32
+; CHECK-NEXT: cmp x10, x8
+; CHECK-NEXT: b.hs .LBB10_1
; CHECK-NEXT: // %bb.2: // %loop.exit
-; CHECK-NEXT: str x9, [x0]
+; CHECK-NEXT: str x10, [x0]
; CHECK-NEXT: ret
entry:
br label %loop.body
@@ -246,14 +247,15 @@ loop.exit:
define void @test_lsr_cbz(ptr %p0) {
; CHECK-LABEL: test_lsr_cbz:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, x0
+; CHECK-NEXT: mov x8, #4294967296 // =0x100000000
+; CHECK-NEXT: mov x9, x0
; CHECK-NEXT: .LBB11_1: // %loop.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldr x9, [x8], #32
-; CHECK-NEXT: lsr x10, x9, #32
-; CHECK-NEXT: cbz x10, .LBB11_1
+; CHECK-NEXT: ldr x10, [x9], #32
+; CHECK-NEXT: cmp x10, x8
+; CHECK-NEXT: b.lo .LBB11_1
; CHECK-NEXT: // %bb.2: // %loop.exit
-; CHECK-NEXT: str x9, [x0]
+; CHECK-NEXT: str x10, [x0]
; CHECK-NEXT: ret
entry:
br label %loop.body
@@ -273,14 +275,15 @@ loop.exit:
define void @test_lshr_icmp0(ptr %p0) {
; CHECK-LABEL: test_lshr_icmp0:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov x8, x0
+; CHECK-NEXT: mov x8, #68719476736 // =0x1000000000
+; CHECK-NEXT: mov x9, x0
; CHECK-NEXT: .LBB12_1: // %loop.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldr x9, [x8], #32
-; CHECK-NEXT: lsr x10, x9, #36
-; CHECK-NEXT: cbnz x10, .LBB12_1
+; CHECK-NEXT: ldr x10, [x9], #32
+; CHECK-NEXT: cmp x10, x8
+; CHECK-NEXT: b.hs .LBB12_1
; CHECK-NEXT: // %bb.2: // %loop.exit
-; CHECK-NEXT: str x9, [x0]
+; CHECK-NEXT: str x10, [x0]
; CHECK-NEXT: ret
entry:
br label %loop.body
diff --git a/llvm/test/CodeGen/AArch64/cond-br-tuning.mir b/llvm/test/CodeGen/AArch64/cond-br-tuning.mir
index 691f88d049b38..8cd0b9c9179b8 100644
--- a/llvm/test/CodeGen/AArch64/cond-br-tuning.mir
+++ b/llvm/test/CodeGen/AArch64/cond-br-tuning.mir
@@ -13,8 +13,9 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 32, 63
- ; CHECK-NEXT: CBNZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 4294967296
+ ; CHECK-NEXT: $xzr = SUBSXrr [[COPY]], [[MOVi64imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 2, %bb.1, implicit $nzcv
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: RET_ReallyLR
@@ -41,8 +42,9 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 47, 63
- ; CHECK-NEXT: CBNZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 140737488355328
+ ; CHECK-NEXT: $xzr = SUBSXrr [[COPY]], [[MOVi64imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 2, %bb.1, implicit $nzcv
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: RET_ReallyLR
@@ -69,8 +71,9 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[UBFMWri:%[0-9]+]]:gpr32 = UBFMWri [[COPY]], 16, 31
- ; CHECK-NEXT: CBZW [[UBFMWri]], %bb.1
+ ; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 65536
+ ; CHECK-NEXT: $wzr = SUBSWrr [[COPY]], [[MOVi32imm]], implicit-def $nzcv
+ ; CHECK-NEXT: Bcc 3, %bb.1, implicit $nzcv
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: RET_ReallyLR
@@ -287,9 +290,10 @@ body: |
; CHECK-NEXT: bb.1:
; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[UBFMXri:%[0-9]+]]:gpr64 = UBFMXri [[COPY]], 32, 63
+ ; CHECK-NEXT: [[MOVi64imm:%[0-9]+]]:gpr64 = MOVi64imm 4294967296
+ ; CHECK-NEXT: $xzr = SUBSXrr [[COPY]], [[MOVi64imm]], implicit-def $nzcv
; CHECK-NEXT: CBNZX [[COPY]], %bb.1
- ; CHECK-NEXT: CBZX [[UBFMXri]], %bb.1
+ ; CHECK-NEXT: Bcc 3, %bb.1, implicit $nzcv
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: RET_ReallyLR
More information about the llvm-commits
mailing list