[llvm] 6b62166 - Account for PATCHABLE instrs in Branch Relaxation
Daniel Hoekwater via llvm-commits
llvm-commits at lists.llvm.org
Fri Apr 14 16:15:25 PDT 2023
Author: Daniel Hoekwater
Date: 2023-04-14T16:14:50-07:00
New Revision: 6b62166b4cf8ec69f40a81d143fa1d623b4b0d0c
URL: https://github.com/llvm/llvm-project/commit/6b62166b4cf8ec69f40a81d143fa1d623b4b0d0c
DIFF: https://github.com/llvm/llvm-project/commit/6b62166b4cf8ec69f40a81d143fa1d623b4b0d0c.diff
LOG: Account for PATCHABLE instrs in Branch Relaxation
PATCHABLE_* instructions expand to up to 36-byte
sleds. Updating the size of PATCHABLE instructions
causes them to be outlined, so we need to add a
check to prevent the outliner from considering
basic blocks that contain PATCHABLE instructions.
Differential Revision: https://reviews.llvm.org/D147982
Added:
llvm/test/CodeGen/AArch64/branch-relax-xray.ll
llvm/test/CodeGen/AArch64/machine-outliner-patchable.mir
Modified:
llvm/lib/CodeGen/TargetInstrInfo.cpp
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 5c5b6a07356d3..9d6c9a6768d5c 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -1651,10 +1651,25 @@ bool TargetInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
// Some instrumentations create special TargetOpcode at the start which
// expands to special code sequences which must be present.
auto First = MBB.getFirstNonDebugInstr();
- if (First != MBB.end() &&
- (First->getOpcode() == TargetOpcode::FENTRY_CALL ||
- First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER))
+ if (First == MBB.end())
+ return true;
+
+ if (First->getOpcode() == TargetOpcode::FENTRY_CALL ||
+ First->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_ENTER)
+ return false;
+
+ // Some instrumentations create special pseudo-instructions at or just before
+ // the end that must be present.
+ auto Last = MBB.getLastNonDebugInstr();
+ if (Last->getOpcode() == TargetOpcode::PATCHABLE_RET ||
+ Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL)
return false;
+ if (Last != First && Last->isReturn()) {
+ --Last;
+ if (Last->getOpcode() == TargetOpcode::PATCHABLE_FUNCTION_EXIT ||
+ Last->getOpcode() == TargetOpcode::PATCHABLE_TAIL_CALL)
+ return false;
+ }
return true;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 69ada004e91ff..0b572e375246f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -126,6 +126,12 @@ unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
if (NumBytes == 0)
NumBytes = 4;
break;
+ case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
+ case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
+ // An XRay sled can be 4 bytes of alignment plus a 32-byte block.
+ NumBytes = 36;
+ break;
+
case AArch64::SPACE:
NumBytes = MI.getOperand(1).getImm();
break;
diff --git a/llvm/test/CodeGen/AArch64/branch-relax-xray.ll b/llvm/test/CodeGen/AArch64/branch-relax-xray.ll
new file mode 100644
index 0000000000000..2458a9d085b00
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/branch-relax-xray.ll
@@ -0,0 +1,42 @@
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu -aarch64-tbz-offset-bits=4 -aarch64-cbz-offset-bits=4 < %s | FileCheck %s
+
+;; Check that branch relaxation accounts for the size of xray EXIT sleds
+;; Note that TAIL_CALL sleds don't exist on AArch64 and don't need a test.
+define void @exit(i1 zeroext %0) nounwind "function-instrument"="xray-always" {
+; CHECK-LABEL: exit:
+; CHECK-NEXT: .Lfunc_begin0:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: .p2align 2
+; CHECK-NEXT: .Lxray_sled_0:
+; CHECK-NEXT: b #32
+; CHECK-COUNT-7: nop
+; CHECK-NOT: nop
+; CHECK: tbnz
+; CHECK-SAME: [[FALLTHROUGH:.LBB[0-9_]+]]
+; CHECK-NEXT: b
+; CHECK-SAME: [[OUT_OF_RANGE:.LBB[0-9_]+]]
+; CHECK-NEXT: [[FALLTHROUGH]]:
+; CHECK-NEXT: bl bar
+; CHECK: .p2align 2
+; CHECK-NEXT: .Lxray_sled_1:
+; CHECK-NEXT: b #32
+; CHECK-COUNT-7: nop
+; CHECK-NOT: nop
+; CHECK-NEXT: .Ltmp1:
+; CHECK-NEXT: ret
+; CHECK-NEXT: [[OUT_OF_RANGE]]:
+; CHECK-SAME: // %end2
+; CHECK-NEXT: bl baz
+ br i1 %0, label %end1, label %end2
+
+end1:
+ %2 = call i32 @bar()
+ ret void
+
+end2:
+ %3 = call i32 @baz()
+ ret void
+}
+
+declare i32 @bar()
+declare i32 @baz()
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll b/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll
index e381b49ee8dc0..f17beedf389a9 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-patchable.ll
@@ -11,7 +11,7 @@ define void @fentry0(i1 %a) nounwind "fentry-call"="true" {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: # FEntry call
; CHECK: // %bb.1:
-; CHECK-NEXT: bl OUTLINED_FUNCTION_1
+; CHECK-NEXT: bl [[OUTLINED_FUNCTION:OUTLINED_FUNCTION_[0-9]+]]
entry:
br i1 %a, label %if.then, label %if.end
if.then:
@@ -27,7 +27,7 @@ define void @fentry1(i1 %a) nounwind "fentry-call"="true" {
; CHECK-NEXT: // %bb.0:
; CHECK-NEXT: # FEntry call
; CHECK: // %bb.1:
-; CHECK-NEXT: bl OUTLINED_FUNCTION_1
+; CHECK-NEXT: bl [[OUTLINED_FUNCTION]]
entry:
br i1 %a, label %if.then, label %if.end
if.then:
@@ -47,7 +47,7 @@ define void @patchable0(i1 %a) nounwind "patchable-function-entry"="2" {
; CHECK-NEXT: nop
; CHECK-NEXT: nop
; CHECK: // %bb.1:
-; CHECK-NEXT: bl OUTLINED_FUNCTION_1
+; CHECK-NEXT: bl [[OUTLINED_FUNCTION]]
entry:
br i1 %a, label %if.then, label %if.end
if.then:
@@ -65,7 +65,7 @@ define void @patchable1(i1 %a) nounwind "patchable-function-entry"="2" {
; CHECK-NEXT: nop
; CHECK-NEXT: nop
; CHECK: // %bb.1:
-; CHECK-NEXT: bl OUTLINED_FUNCTION_1
+; CHECK-NEXT: bl [[OUTLINED_FUNCTION]]
entry:
br i1 %a, label %if.then, label %if.end
if.then:
@@ -84,7 +84,7 @@ define void @xray0(i1 %a) nounwind "function-instrument"="xray-always" {
; CHECK-NEXT: .p2align 2
; CHECK-NEXT: .Lxray_sled_0:
; CHECK: // %bb.1:
-; CHECK-NEXT: bl OUTLINED_FUNCTION_1
+; CHECK-NEXT: bl [[OUTLINED_FUNCTION]]
entry:
br i1 %a, label %if.then, label %if.end
if.then:
@@ -102,7 +102,7 @@ define void @xray1(i1 %a) nounwind "function-instrument"="xray-always" {
; CHECK-NEXT: .p2align 2
; CHECK-NEXT: .Lxray_sled_2:
; CHECK: // %bb.1:
-; CHECK-NEXT: bl OUTLINED_FUNCTION_1
+; CHECK-NEXT: bl [[OUTLINED_FUNCTION]]
entry:
br i1 %a, label %if.then, label %if.end
if.then:
@@ -112,3 +112,12 @@ if.end:
call void @foo(i32 5, i32 6, i32 7, i32 8)
ret void
}
+
+;; Make sure that OUTLINED_FUNCTION contains the right instructions
+; CHECK: [[OUTLINED_FUNCTION]]:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: mov w0, #1
+; CHECK-NEXT: mov w1, #2
+; CHECK-NEXT: mov w2, #3
+; CHECK-NEXT: mov w3, #4
+; CHECK-NEXT: b foo
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-patchable.mir b/llvm/test/CodeGen/AArch64/machine-outliner-patchable.mir
new file mode 100644
index 0000000000000..6e957bceb8f4f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-patchable.mir
@@ -0,0 +1,162 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass machine-outliner -verify-machineinstrs -enable-machine-outliner %s -o - | FileCheck %s
+--- |
+ ; Function Attrs: minsize
+ declare void @foo(i32, i32, i32, i32) #0
+
+ ; Function Attrs: nounwind
+ define void @xray0(i1 %a) #1 {
+ entry:
+ br i1 %a, label %if.then, label %if.end
+
+ if.then: ; preds = %entry
+ call void @foo(i32 1, i32 2, i32 3, i32 4)
+ br label %if.end
+
+ if.end: ; preds = %if.then, %entry
+ call void @foo(i32 5, i32 6, i32 7, i32 8)
+ ret void
+ }
+
+ ; Function Attrs: nounwind
+ define void @xray1(i1 %a) #1 {
+ entry:
+ br i1 %a, label %if.then, label %if.end
+
+ if.then: ; preds = %entry
+ call void @foo(i32 1, i32 2, i32 3, i32 4)
+ br label %if.end
+
+ if.end: ; preds = %if.then, %entry
+ call void @foo(i32 5, i32 6, i32 7, i32 8)
+ ret void
+ }
+
+ attributes #0 = { minsize }
+ attributes #1 = { nounwind "function-instrument"="xray-always" }
+
+...
+---
+name: xray0
+tracksRegLiveness: true
+liveins:
+ - { reg: '$w0', virtual-reg: '' }
+stack:
+ - { id: 0, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16,
+ stack-id: default, callee-saved-register: '$lr', callee-saved-restored: true,
+ debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ ; CHECK-LABEL: name: xray0
+ ; CHECK: bb.0.entry:
+ ; CHECK: PATCHABLE_FUNCTION_ENTER
+ ; CHECK: bb.1.if.then:
+ ; CHECK: BL @[[OUTLINED_FUNCTION:OUTLINED_FUNCTION_[0-9]]]
+ ; CHECK: bb.2.if.end:
+ ; CHECK-NEXT: $w0 = MOVZWi 5, 0
+ ; CHECK-NEXT: $w1 = MOVZWi 6, 0
+ ; CHECK-NEXT: $w2 = MOVZWi 7, 0
+ ; CHECK-NEXT: $w3 = MOVZWi 8, 0
+ ; CHECK-NEXT: BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp
+ ; CHECK: $w0 = MOVZWi 5, 0
+ ; CHECK-NEXT: $w1 = MOVZWi 6, 0
+ ; CHECK-NEXT: PATCHABLE_FUNCTION_EXIT
+ ; CHECK-NEXT: RET undef $lr
+
+ bb.0.entry:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $w0, $lr
+
+ PATCHABLE_FUNCTION_ENTER
+ early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.0)
+ TBZW killed renamable $w0, 0, %bb.2
+
+ bb.1.if.then:
+ successors: %bb.2(0x80000000)
+
+ $w0 = MOVZWi 1, 0
+ $w1 = MOVZWi 2, 0
+ $w2 = MOVZWi 3, 0
+ $w3 = MOVZWi 4, 0
+ BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp
+
+ bb.2.if.end:
+ $w0 = MOVZWi 5, 0
+ $w1 = MOVZWi 6, 0
+ $w2 = MOVZWi 7, 0
+ $w3 = MOVZWi 8, 0
+ BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp
+ early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0)
+ $w0 = MOVZWi 5, 0
+ $w1 = MOVZWi 6, 0
+ PATCHABLE_FUNCTION_EXIT
+ RET undef $lr
+
+...
+---
+name: xray1
+tracksRegLiveness: true
+liveins:
+ - { reg: '$w0', virtual-reg: '' }
+stack:
+ - { id: 0, name: '', type: spill-slot, offset: -16, size: 8, alignment: 16,
+ stack-id: default, callee-saved-register: '$lr', callee-saved-restored: true,
+ debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+machineFunctionInfo:
+ hasRedZone: false
+body: |
+ ; CHECK-LABEL: name: xray1
+ ; CHECK: bb.0.entry:
+ ; CHECK: PATCHABLE_FUNCTION_ENTER
+ ; CHECK: bb.1.if.then:
+ ; CHECK: BL @[[OUTLINED_FUNCTION]]
+ ; CHECK: bb.2.if.end:
+ ; CHECK-NEXT: $w0 = MOVZWi 5, 0
+ ; CHECK-NEXT: $w1 = MOVZWi 6, 0
+ ; CHECK-NEXT: $w2 = MOVZWi 7, 0
+ ; CHECK-NEXT: $w3 = MOVZWi 8, 0
+ ; CHECK-NEXT: BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp
+ ; CHECK: $w0 = MOVZWi 5, 0
+ ; CHECK-NEXT: $w1 = MOVZWi 6, 0
+ ; CHECK-NEXT: PATCHABLE_FUNCTION_EXIT
+ ; CHECK-NEXT: RET undef $lr
+
+ bb.0.entry:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ liveins: $w0, $lr
+
+ PATCHABLE_FUNCTION_ENTER
+ early-clobber $sp = frame-setup STRXpre killed $lr, $sp, -16 :: (store (s64) into %stack.0)
+ TBZW killed renamable $w0, 0, %bb.2
+
+ bb.1.if.then:
+ successors: %bb.2(0x80000000)
+
+ $w0 = MOVZWi 1, 0
+ $w1 = MOVZWi 2, 0
+ $w2 = MOVZWi 3, 0
+ $w3 = MOVZWi 4, 0
+ BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp
+
+ bb.2.if.end:
+ $w0 = MOVZWi 5, 0
+ $w1 = MOVZWi 6, 0
+ $w2 = MOVZWi 7, 0
+ $w3 = MOVZWi 8, 0
+ BL @foo, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $w0, implicit killed $w1, implicit killed $w2, implicit killed $w3, implicit-def $sp
+ early-clobber $sp, $lr = frame-destroy LDRXpost $sp, 16 :: (load (s64) from %stack.0)
+ $w0 = MOVZWi 5, 0
+ $w1 = MOVZWi 6, 0
+ PATCHABLE_FUNCTION_EXIT
+ RET undef $lr
+
+ ; CHECK: name: [[OUTLINED_FUNCTION]]
+ ; CHECK: bb.0:
+ ; CHECK: $w0 = MOVZWi 1, 0
+ ; CHECK-NEXT: $w1 = MOVZWi 2, 0
+ ; CHECK-NEXT: $w2 = MOVZWi 3, 0
+ ; CHECK-NEXT: $w3 = MOVZWi 4, 0
+ ; CHECK-NEXT: TCRETURNdi @foo, 0, implicit $sp
+
+...
+
More information about the llvm-commits
mailing list