[llvm-branch-commits] [llvm] [AArch64][MachineOutliner] Handle async CFI for outlined calls (PR #236836)

Csanád Hajdú via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Thu Oct 29 09:46:07 PDT 2026


https://github.com/Il-Capitano updated https://github.com/llvm/llvm-project/pull/236836

>From 0c08aa0121b5f783cb4284cd91387183ecc3a03f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Csan=C3=A1d=20Hajd=C3=BA?= <csanad.hajdu at arm.com>
Date: Wed, 28 Oct 2026 10:37:39 +0000
Subject: [PATCH 1/2] [AArch64][MachineOutliner] Emit async CFI in outlined
 functions

When requesting asynchronous unwind info, CFI instructions are now added
to the epilogue of outlined functions. The prologue CFI is also modified
to make it compatible with asynchronous unwinding.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  | 38 ++++++++++---
 .../AArch64/machine-outliner-calls.mir        |  9 ++--
 ...outliner-macho-compact-unwind-frame-fp.mir |  9 +++-
 ...ne-outliner-macho-compact-unwind-frame.mir | 53 +++++++++++++++++--
 .../machine-outliner-sp-addsub-fixup.mir      |  8 +++
 .../AArch64/machine-outliner-throw2.ll        |  4 +-
 6 files changed, 103 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 05f356308fe08e..a9781e82da116a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -11552,14 +11552,20 @@ void AArch64InstrInfo::buildOutlinedFrame(
                                 .addImm(0);
       MBB.insertAfter(It, SetFP);
 
+      CFIInstBuilder SetupCFIBuilder(MBB, std::next(STPXpre->getIterator()),
+                                     MachineInstr::FrameSetup);
+      // CFA needs to be adjusted between the STP and MOV when async unwind info
+      // is needed.
+      if (FI->needsAsyncDwarfUnwindInfo(MF))
+        SetupCFIBuilder.buildDefCFAOffset(16);
+
       // Describe the frame record with FP as the CFA. The encoder needs all
       // three to pick FRAME. No need to check for unwind info here: we only
       // get here if the function has it.
-      CFIInstBuilder CFIBuilder(MBB, std::next(SetFP->getIterator()),
-                                MachineInstr::FrameSetup);
-      CFIBuilder.buildDefCFA(AArch64::FP, 16);
-      CFIBuilder.buildOffset(AArch64::LR, -8);
-      CFIBuilder.buildOffset(AArch64::FP, -16);
+      SetupCFIBuilder.setInsertPoint(std::next(SetFP->getIterator()));
+      SetupCFIBuilder.buildDefCFA(AArch64::FP, 16);
+      SetupCFIBuilder.buildOffset(AArch64::LR, -8);
+      SetupCFIBuilder.buildOffset(AArch64::FP, -16);
 
       // ldp x29, x30, [sp], #16
       MachineInstr *LDPXpost = BuildMI(MF, DebugLoc(), get(AArch64::LDPXpost))
@@ -11569,6 +11575,16 @@ void AArch64InstrInfo::buildOutlinedFrame(
                                    .addReg(AArch64::SP)
                                    .addImm(2);
       Et = MBB.insert(Et, LDPXpost);
+
+      // Add the necessary CFI instructions to the epilogue if they are needed.
+      if (FI->needsAsyncDwarfUnwindInfo(MF)) {
+        CFIInstBuilder DestroyCFIBuilder(MBB, Et, MachineInstr::FrameDestroy);
+        DestroyCFIBuilder.buildDefCFA(AArch64::SP, 16);
+        DestroyCFIBuilder.setInsertPoint(std::next(Et));
+        DestroyCFIBuilder.buildDefCFAOffset(0);
+        DestroyCFIBuilder.buildRestore(AArch64::LR);
+        DestroyCFIBuilder.buildRestore(AArch64::FP);
+      }
     } else {
       // Insert a save before the outlined region
       MachineInstr *STRXpre = BuildMI(MF, DebugLoc(), get(AArch64::STRXpre))
@@ -11578,8 +11594,8 @@ void AArch64InstrInfo::buildOutlinedFrame(
                                   .addImm(-16);
       It = MBB.insert(It, STRXpre);
 
-      if (MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
-        CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
+      if (FI->needsDwarfUnwindInfo(MF)) {
+        CFIInstBuilder CFIBuilder(MBB, std::next(It), MachineInstr::FrameSetup);
 
         // Add a CFI saying the stack was moved 16 B down.
         CFIBuilder.buildDefCFAOffset(16);
@@ -11596,6 +11612,14 @@ void AArch64InstrInfo::buildOutlinedFrame(
                                    .addReg(AArch64::SP)
                                    .addImm(16);
       Et = MBB.insert(Et, LDRXpost);
+
+      // Add the necessary CFI instructions to the epilogue if they are needed.
+      if (FI->needsAsyncDwarfUnwindInfo(MF)) {
+        CFIInstBuilder CFIBuilder(MBB, std::next(Et),
+                                  MachineInstr::FrameDestroy);
+        CFIBuilder.buildDefCFAOffset(0);
+        CFIBuilder.buildRestore(AArch64::LR);
+      }
     }
   }
 
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-calls.mir b/llvm/test/CodeGen/AArch64/machine-outliner-calls.mir
index eb858d532c51cf..1853ace4df3a01 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-calls.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-calls.mir
@@ -1,5 +1,6 @@
 # RUN: llc -mtriple=aarch64--- -run-pass=prolog-epilog -run-pass=machine-outliner -verify-machineinstrs %s -o - | FileCheck %s
 # RUN: llc -mtriple=aarch64--- -run-pass=prolog-epilog -run-pass=machine-outliner -verify-machineinstrs %s -o - -debugify-and-strip-all-safe | FileCheck %s
+# RUN: llc -mtriple=aarch64--- -run-pass=prolog-epilog -run-pass=machine-outliner -verify-machineinstrs -aarch64-enable-async-unwind-for-minsize %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-ASYNC
 # RUN: llc -mtriple=aarch64-pc-windows-msvc -run-pass=prolog-epilog -run-pass=machine-outliner -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=WINDOWS
 --- |
   define void @baz() #0 {
@@ -10,7 +11,7 @@
     ret void
   }
   
-  attributes #0 = { noredzone }
+  attributes #0 = { noredzone uwtable }
 ...
 ---
 
@@ -59,12 +60,14 @@ body:             |
 # CHECK: name:            OUTLINED_FUNCTION_0
 # CHECK: bb.0:
 # CHECK: liveins: $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $d8, $d9, $d10, $d11, $d12, $d13, $d14, $d15, $lr
-# CHECK-DAG: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-DAG: early-clobber $sp = STRXpre $lr, $sp, -16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16
-# CHECK-NEXT: early-clobber $sp = STRXpre $lr, $sp, -16
 # CHECK-NEXT: BL @baz, implicit-def dead $lr, implicit $sp
 # CHECK-NEXT: $w17 = ORRWri $wzr, 1
 # CHECK-NEXT: $w17 = ORRWri $wzr, 1
 # CHECK-NEXT: early-clobber $sp, $lr = LDRXpost $sp, 16
+# CHECK-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION restore $w30
 # CHECK-NEXT: RET $lr
 # WINDOWS-NOT: OUTLINED
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-macho-compact-unwind-frame-fp.mir b/llvm/test/CodeGen/AArch64/machine-outliner-macho-compact-unwind-frame-fp.mir
index 4187c6ad308e50..02e58853948d22 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-macho-compact-unwind-frame-fp.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-macho-compact-unwind-frame-fp.mir
@@ -1,5 +1,8 @@
 # RUN: llc -mtriple=arm64-apple-darwin -run-pass=prolog-epilog -run-pass=machine-outliner \
 # RUN:     -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=arm64-apple-darwin -run-pass=prolog-epilog -run-pass=machine-outliner \
+# RUN:     -verify-machineinstrs -aarch64-enable-async-unwind-for-minsize %s -o - | \
+# RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-ASYNC
 
 # Do not use the MachO compact-unwind frame-record form when the outlined
 # sequence itself reads or writes FP. Restoring FP in the outlined epilogue
@@ -77,13 +80,15 @@ body:             |
 # CHECK: name:            OUTLINED_FUNCTION_0
 # CHECK-NOT: STPXpre
 # CHECK-NOT: $fp = ADDXri $sp, 0, 0
-# CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK: early-clobber $sp = STRXpre $lr, $sp, -16
+# CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
 # CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16
-# CHECK-NEXT: early-clobber $sp = STRXpre $lr, $sp, -16
 # CHECK-NEXT: BL @baz, implicit-def dead $lr, implicit $sp
 # CHECK-NEXT: $x9 = ORRXrs $xzr, $fp, 0
 # CHECK-NEXT: $fp = ORRXrs $xzr, $x9, 0
 # CHECK-NEXT: BL @baz, implicit-def dead $lr, implicit $sp
 # CHECK-NEXT: $w17 = ORRWri $wzr, 1
 # CHECK: early-clobber $sp, $lr = LDRXpost $sp, 16
+# CHECK-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# CHECK-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION restore $w30
 # CHECK-NEXT: RET $lr
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-macho-compact-unwind-frame.mir b/llvm/test/CodeGen/AArch64/machine-outliner-macho-compact-unwind-frame.mir
index 6d6dd6a9404464..e4e45d55b78294 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-macho-compact-unwind-frame.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-macho-compact-unwind-frame.mir
@@ -1,5 +1,8 @@
 # RUN: llc -mtriple=arm64-apple-darwin -run-pass=prolog-epilog -run-pass=machine-outliner \
 # RUN:     -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=MACHO
+# RUN: llc -mtriple=arm64-apple-darwin -run-pass=prolog-epilog -run-pass=machine-outliner \
+# RUN:     -verify-machineinstrs -aarch64-enable-async-unwind-for-minsize %s -o - | \
+# RUN:     FileCheck %s --check-prefix=MACHO-ASYNC
 # RUN: llc -mtriple=arm64-apple-darwin \
 # RUN:     -run-pass=mir-strip-debug,prolog-epilog,machine-outliner \
 # RUN:     -mir-strip-debugify-only=0 -verify-machineinstrs %s -o - | \
@@ -7,8 +10,15 @@
 # RUN: llc -mtriple=arm64-apple-darwin -run-pass=prolog-epilog -run-pass=machine-outliner \
 # RUN:     -aarch64-outliner-compact-unwind-frame=false -verify-machineinstrs %s -o - | \
 # RUN:     FileCheck %s --check-prefix=NOFRAME
+# RUN: llc -mtriple=arm64-apple-darwin -run-pass=prolog-epilog -run-pass=machine-outliner \
+# RUN:     -aarch64-outliner-compact-unwind-frame=false -verify-machineinstrs \
+# RUN:     -aarch64-enable-async-unwind-for-minsize %s -o - | \
+# RUN:     FileCheck %s --check-prefix=NOFRAME-ASYNC
 # RUN: llc -mtriple=aarch64-linux-gnu -run-pass=prolog-epilog -run-pass=machine-outliner \
 # RUN:     -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=ELF
+# RUN: llc -mtriple=aarch64-linux-gnu -run-pass=prolog-epilog -run-pass=machine-outliner \
+# RUN:     -verify-machineinstrs -aarch64-enable-async-unwind-for-minsize %s -o - | \
+# RUN:     FileCheck %s --check-prefix=ELF-ASYNC
 
 # A non-leaf outlined function must save LR. On MachO, save it as a full frame
 # record (stp x29,x30 ; mov x29,sp) so the compact-unwind encoder can use the
@@ -105,21 +115,56 @@ body:             |
 # MACHO: early-clobber $sp, $fp, $lr = LDPXpost $sp, 2
 # MACHO-NEXT: RET $lr
 
+# MACHO-ASYNC: name:            OUTLINED_FUNCTION_0
+# MACHO-ASYNC: early-clobber $sp = STPXpre $fp, $lr, $sp, -2
+# MACHO-ASYNC-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# MACHO-ASYNC-NEXT: $fp = ADDXri $sp, 0, 0
+# MACHO-ASYNC-NEXT: frame-setup CFI_INSTRUCTION def_cfa $w29, 16
+# MACHO-ASYNC-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -8
+# MACHO-ASYNC-NEXT: frame-setup CFI_INSTRUCTION offset $w29, -16
+# MACHO-ASYNC-NEXT: BL @baz, implicit-def dead $lr, implicit $sp
+# MACHO-ASYNC: early-clobber $sp, $fp, $lr = LDPXpost $sp, 2
+# MACHO-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# MACHO-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION restore $w30
+# MACHO-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION restore $w29
+# MACHO-ASYNC-NEXT: RET $lr
+
 # With the frame-record optimization disabled, MachO keeps the LR-alone save.
 # NOFRAME: name:            OUTLINED_FUNCTION_0
 # NOFRAME-NOT: STPXpre
-# NOFRAME: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# NOFRAME: early-clobber $sp = STRXpre $lr, $sp, -16
+# NOFRAME-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
 # NOFRAME-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16
-# NOFRAME-NEXT: early-clobber $sp = STRXpre $lr, $sp, -16
 # NOFRAME-NEXT: BL @baz, implicit-def dead $lr, implicit $sp
 # NOFRAME: early-clobber $sp, $lr = LDRXpost $sp, 16
 # NOFRAME-NEXT: RET $lr
 
+# NOFRAME-ASYNC: name:            OUTLINED_FUNCTION_0
+# NOFRAME-ASYNC-NOT: STPXpre
+# NOFRAME-ASYNC: early-clobber $sp = STRXpre $lr, $sp, -16
+# NOFRAME-ASYNC-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# NOFRAME-ASYNC-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16
+# NOFRAME-ASYNC-NEXT: BL @baz, implicit-def dead $lr, implicit $sp
+# NOFRAME-ASYNC: early-clobber $sp, $lr = LDRXpost $sp, 16
+# NOFRAME-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# NOFRAME-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION restore $w30
+# NOFRAME-ASYNC-NEXT: RET $lr
+
 # On ELF the LR-alone save is unchanged.
 # ELF: name:            OUTLINED_FUNCTION_0
-# ELF: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# ELF: early-clobber $sp = STRXpre $lr, $sp, -16
+# ELF-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
 # ELF-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16
-# ELF-NEXT: early-clobber $sp = STRXpre $lr, $sp, -16
 # ELF-NEXT: BL @baz, implicit-def dead $lr, implicit $sp
 # ELF: early-clobber $sp, $lr = LDRXpost $sp, 16
 # ELF-NEXT: RET $lr
+
+# ELF-ASYNC: name:            OUTLINED_FUNCTION_0
+# ELF-ASYNC: early-clobber $sp = STRXpre $lr, $sp, -16
+# ELF-ASYNC-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# ELF-ASYNC-NEXT: frame-setup CFI_INSTRUCTION offset $w30, -16
+# ELF-ASYNC-NEXT: BL @baz, implicit-def dead $lr, implicit $sp
+# ELF-ASYNC: early-clobber $sp, $lr = LDRXpost $sp, 16
+# ELF-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION def_cfa_offset 0
+# ELF-ASYNC-NEXT: frame-destroy CFI_INSTRUCTION restore $w30
+# ELF-ASYNC-NEXT: RET $lr
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-sp-addsub-fixup.mir b/llvm/test/CodeGen/AArch64/machine-outliner-sp-addsub-fixup.mir
index 4d1b7586f43f3a..1dac1543c4edca 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-sp-addsub-fixup.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-sp-addsub-fixup.mir
@@ -588,6 +588,8 @@ body:             |
 
 # CHECK:       name:            [[ADD_FUNC]]
 # CHECK:       early-clobber $sp = STRXpre $lr, $sp, -16
+# CHECK-NEXT:  frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT:  frame-setup CFI_INSTRUCTION offset $w30, -16
 # CHECK-NEXT:  BL @callee
 # CHECK-NEXT:  $x15 = ADDXri $sp, 64, 0
 # CHECK-NEXT:  $x15 = ADDXri $sp, 64, 0
@@ -596,6 +598,8 @@ body:             |
 
 # CHECK:       name:            [[SUB_FUNC]]
 # CHECK:       early-clobber $sp = STRXpre $lr, $sp, -16
+# CHECK-NEXT:  frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT:  frame-setup CFI_INSTRUCTION offset $w30, -16
 # CHECK-NEXT:  BL @callee
 # CHECK-NEXT:  $x15 = SUBXri $sp, 32, 0
 # CHECK-NEXT:  $x15 = SUBXri $sp, 32, 0
@@ -604,6 +608,8 @@ body:             |
 
 # CHECK:       name:            [[SHIFTED_SUB_FUNC]]
 # CHECK:       early-clobber $sp = STRXpre $lr, $sp, -16
+# CHECK-NEXT:  frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT:  frame-setup CFI_INSTRUCTION offset $w30, -16
 # CHECK-NEXT:  BL @callee
 # CHECK-NEXT:  $x15 = SUBXri $sp, 4080, 0
 # CHECK-NEXT:  $x15 = SUBXri $sp, 4080, 0
@@ -612,6 +618,8 @@ body:             |
 
 # CHECK:       name:            [[SUB_TO_ADD_FUNC]]
 # CHECK:       early-clobber $sp = STRXpre $lr, $sp, -16
+# CHECK-NEXT:  frame-setup CFI_INSTRUCTION def_cfa_offset 16
+# CHECK-NEXT:  frame-setup CFI_INSTRUCTION offset $w30, -16
 # CHECK-NEXT:  BL @callee
 # CHECK-NEXT:  $x15 = ADDXri $sp, 8, 0
 # CHECK-NEXT:  $x15 = ADDXri $sp, 8, 0
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll b/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll
index 538e1165e39c1d..62ecec3ab29db1 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll
@@ -71,9 +71,9 @@ entry:
 ; OMITFP-LABEL: OUTLINED_FUNCTION_0:
 ; OMITFP:              .cfi_startproc
 ; OMITFP-NEXT: // %bb.0:
+; OMITFP-NEXT:         str     x30, [sp, #-16]!
 ; OMITFP-NEXT:         .cfi_def_cfa_offset 16
 ; OMITFP-NEXT:         .cfi_offset w30, -16
-; OMITFP-NEXT:         str     x30, [sp, #-16]!
 ; OMITFP-NEXT:         mov     w0, #1
 ; OMITFP-NEXT:         mov     w1, #2
 ; OMITFP-NEXT:         mov     w2, #3
@@ -87,9 +87,9 @@ entry:
 ; NOOMIT-LABEL: OUTLINED_FUNCTION_0:
 ; NOOMIT:              .cfi_startproc
 ; NOOMIT-NEXT: // %bb.0:
+; NOOMIT-NEXT:         str     x30, [sp, #-16]!
 ; NOOMIT-NEXT:         .cfi_def_cfa_offset 16
 ; NOOMIT-NEXT:         .cfi_offset w30, -16
-; NOOMIT-NEXT:         str     x30, [sp, #-16]!
 ; NOOMIT-NEXT:         mov     w0, #1
 ; NOOMIT-NEXT:         mov     w1, #2
 ; NOOMIT-NEXT:         mov     w2, #3

>From 20f520bd5a2e66877a35062da76914da02b6f662 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Csan=C3=A1d=20Hajd=C3=BA?= <csanad.hajdu at arm.com>
Date: Wed, 28 Oct 2026 14:20:55 +0000
Subject: [PATCH 2/2] [AArch64][MachineOutliner] Handle async CFI for outlined
 calls

When calling outlined functions from a function that needs asynchronous
unwind information, we need to add extra CFI instructions at the call
site in some cases, depending on how LR is saved in the caller. We have
the following cases:

1. LR doesn't need saving at the call site. In this case we don't need
   any extra CFI instructions.
2. LR is saved in a free register. In this case we need to add
   `.cfi_register w30, wN` and `.cfi_restore w30` around the function
   call. This will make sure that an unwinder can recover the correct LR
   value.
3. LR is saved to the stack. Since this case modifies the value of SP,
   and thus potentially the CFA, we disallow it entirely with
   asynchronous unwind info. We could probably allow some special cases,
   e.g. when the function doesn't use FP, but then we would have to
   reimplement complex functionality from `AArch64FrameLowering`.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp  | 96 ++++++++++++++-----
 ...machine-outliner-compatible-candidates.mir | 35 ++++---
 .../AArch64/machine-outliner-drop-stack.mir   | 24 +++--
 .../AArch64/machine-outliner-regsave.mir      | 38 +++++---
 4 files changed, 139 insertions(+), 54 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index a9781e82da116a..25e656476c30f9 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -11048,6 +11048,27 @@ AArch64InstrInfo::getOutliningCandidateInfo(
       }
     }
 
+    // Disallow saving LR to the stack if async unwind information is needed,
+    // because it's difficult to determine what CFI instructions are required
+    // for correct unwind information.
+    // TODO: We could probably allow this in some special cases, e.g. if the
+    // function doesn't use FP, we can emit CFI similar to the following:
+    //
+    //   str x30, [sp, #-16]!
+    //   .cfi_adjust_cfa_offset 16
+    //   .cfi_rel_offset w30, 0
+    //   bl OUTLINED_FUNCTION_N
+    //   ldr x30, [sp], #16
+    //   .cfi_adjust_cfa_offset -16
+    //   .cfi_restore w30
+    //
+    llvm::erase_if(RepeatedSequenceLocs, [](outliner::Candidate &C) {
+      return C.CallConstructionID == MachineOutlinerDefault &&
+             C.getMF()
+                 ->getInfo<AArch64FunctionInfo>()
+                 ->needsAsyncDwarfUnwindInfo(*C.getMF());
+    });
+
     // If we dropped all of the candidates, bail out here.
     if (RepeatedSequenceLocs.size() < MinRepeats)
       return std::nullopt;
@@ -11660,6 +11681,7 @@ void AArch64InstrInfo::buildOutlinedFrame(
 MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
     Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
     MachineFunction &MF, outliner::Candidate &C) const {
+  MachineFunction &CallerMF = *MBB.getParent();
 
   // Are we tail calling?
   if (C.CallConstructionID == MachineOutlinerTailCall) {
@@ -11684,8 +11706,8 @@ MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
 
   // Instructions for saving and restoring LR around the call instruction we're
   // going to insert.
-  MachineInstr *Save;
-  MachineInstr *Restore;
+  SmallVector<MachineInstr *, 2> SaveInstrs;
+  SmallVector<MachineInstr *, 2> RestoreInstrs;
   // Can we save to a register?
   if (C.CallConstructionID == MachineOutlinerRegSave) {
     // FIXME: This logic should be sunk into a target-specific interface so that
@@ -11697,39 +11719,65 @@ MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
     if (!MBB.isLiveIn(AArch64::LR))
       MBB.addLiveIn(AArch64::LR);
 
+    AArch64FunctionInfo *MFI = CallerMF.getInfo<AArch64FunctionInfo>();
+    bool NeedsAsyncDwarfUnwindInfo = MFI->needsAsyncDwarfUnwindInfo(CallerMF);
+    unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true);
+    unsigned DwarfLR = TRI.getDwarfRegNum(AArch64::LR, true);
+
     // Save and restore LR from Reg.
-    Save = BuildMI(MF, DebugLoc(), get(AArch64::ORRXrs), Reg)
-               .addReg(AArch64::XZR)
-               .addReg(AArch64::LR)
-               .addImm(0);
-    Restore = BuildMI(MF, DebugLoc(), get(AArch64::ORRXrs), AArch64::LR)
-                .addReg(AArch64::XZR)
-                .addReg(Reg)
-                .addImm(0);
+    SaveInstrs.push_back(
+        BuildMI(CallerMF, DebugLoc(), get(AArch64::ORRXrs), Reg)
+            .addReg(AArch64::XZR)
+            .addReg(AArch64::LR)
+            .addImm(0));
+    if (NeedsAsyncDwarfUnwindInfo) {
+      unsigned Index = CallerMF.addFrameInst(
+          MCCFIInstruction::createRegister(nullptr, DwarfLR, DwarfReg));
+      SaveInstrs.push_back(
+          BuildMI(CallerMF, DebugLoc(), get(TargetOpcode::CFI_INSTRUCTION))
+              .addCFIIndex(Index));
+    }
+
+    RestoreInstrs.push_back(
+        BuildMI(CallerMF, DebugLoc(), get(AArch64::ORRXrs), AArch64::LR)
+            .addReg(AArch64::XZR)
+            .addReg(Reg)
+            .addImm(0));
+    if (NeedsAsyncDwarfUnwindInfo) {
+      unsigned Index = CallerMF.addFrameInst(
+          MCCFIInstruction::createRestore(nullptr, DwarfLR));
+      RestoreInstrs.push_back(
+          BuildMI(CallerMF, DebugLoc(), get(TargetOpcode::CFI_INSTRUCTION))
+              .addCFIIndex(Index));
+    }
   } else {
     // We have the default case. Save and restore from SP.
-    Save = BuildMI(MF, DebugLoc(), get(AArch64::STRXpre))
-               .addReg(AArch64::SP, RegState::Define)
-               .addReg(AArch64::LR)
-               .addReg(AArch64::SP)
-               .addImm(-16);
-    Restore = BuildMI(MF, DebugLoc(), get(AArch64::LDRXpost))
-                  .addReg(AArch64::SP, RegState::Define)
-                  .addReg(AArch64::LR, RegState::Define)
-                  .addReg(AArch64::SP)
-                  .addImm(16);
+    SaveInstrs.push_back(BuildMI(CallerMF, DebugLoc(), get(AArch64::STRXpre))
+                             .addReg(AArch64::SP, RegState::Define)
+                             .addReg(AArch64::LR)
+                             .addReg(AArch64::SP)
+                             .addImm(-16));
+    RestoreInstrs.push_back(
+        BuildMI(CallerMF, DebugLoc(), get(AArch64::LDRXpost))
+            .addReg(AArch64::SP, RegState::Define)
+            .addReg(AArch64::LR, RegState::Define)
+            .addReg(AArch64::SP)
+            .addImm(16));
   }
 
-  It = MBB.insert(It, Save);
-  It++;
+  for (MachineInstr *Save : SaveInstrs) {
+    It = MBB.insert(It, Save);
+    ++It;
+  }
 
   // Insert the call.
   It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(AArch64::BL))
                           .addGlobalAddress(M.getNamedValue(MF.getName())));
   CallPt = It;
-  It++;
 
-  It = MBB.insert(It, Restore);
+  for (MachineInstr *Restore : RestoreInstrs)
+    It = MBB.insert(std::next(It), Restore);
+
   return CallPt;
 }
 
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-compatible-candidates.mir b/llvm/test/CodeGen/AArch64/machine-outliner-compatible-candidates.mir
index 9f2fa7ca1888b2..1659fec911f905 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-compatible-candidates.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-compatible-candidates.mir
@@ -1,5 +1,8 @@
-# RUN: llc -mtriple=aarch64--- -run-pass=machine-outliner \
-# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -run-pass=machine-outliner -verify-machineinstrs %s -o - | \
+# RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-SYNC
+# RUN: llc -mtriple=aarch64 -run-pass=machine-outliner -verify-machineinstrs \
+# RUN:     -aarch64-enable-async-unwind-for-minsize %s -o - | \
+# RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-ASYNC
 
 # Ensure that we can outline candidates with compatible call/frame classes.
 #
@@ -12,7 +15,7 @@
   define void @no-save2() #0 { ret void }
   define void @reg-save() #0 { ret void }
   define void @stack-save() #0 { ret void }
-  attributes #0 = { minsize noinline noredzone "frame-pointer"="all" }
+  attributes #0 = { minsize noinline noredzone "frame-pointer"="all" uwtable }
 ...
 ---
 
@@ -67,10 +70,17 @@ body:             |
   $lr = ORRXri $xzr, 1
   bb.1:
   liveins: $lr
-    ; CHECK-LABEL: name:            reg-save
-    ; CHECK: $[[REG:x[0-9]+]] = ORRXrs $xzr, $lr, 0
-    ; CHECK-NEXT: BL [[FN]]
-    ; CHECK-NEXT: $lr = ORRXrs $xzr, $[[REG]], 0
+    ; CHECK-SYNC-LABEL: name:            reg-save
+    ; CHECK-SYNC: $x[[REG:[0-9]+]] = ORRXrs $xzr, $lr, 0
+    ; CHECK-SYNC-NEXT: BL [[FN]]
+    ; CHECK-SYNC-NEXT: $lr = ORRXrs $xzr, $x[[REG]], 0
+    ;
+    ; CHECK-ASYNC-LABEL: name:            reg-save
+    ; CHECK-ASYNC: $x[[REG:[0-9]+]] = ORRXrs $xzr, $lr, 0
+    ; CHECK-ASYNC-NEXT: CFI_INSTRUCTION register $w30, $w[[REG]]
+    ; CHECK-ASYNC-NEXT: BL [[FN]]
+    ; CHECK-ASYNC-NEXT: $lr = ORRXrs $xzr, $x[[REG]], 0
+    ; CHECK-ASYNC-NEXT: CFI_INSTRUCTION restore $w30
     $w11 = ORRWri $wzr, 1
     $w11 = ORRWri $wzr, 1
     $w11 = ORRWri $wzr, 1
@@ -90,10 +100,13 @@ body:             |
   $lr = ORRXri $xzr, 1
   bb.1:
   liveins: $lr, $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x18, $x19, $x20, $x21, $x22, $x23, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $fp
-    ; CHECK-LABEL: name:            stack-save
-    ; CHECK: $sp = STRXpre $lr, $sp, -16
-    ; CHECK-NEXT: BL [[FN]]
-    ; CHECK-NEXT: $sp, $lr = LDRXpost $sp, 16
+    ; CHECK-SYNC-LABEL: name:            stack-save
+    ; CHECK-SYNC: $sp = STRXpre $lr, $sp, -16
+    ; CHECK-SYNC-NEXT: BL [[FN]]
+    ; CHECK-SYNC-NEXT: $sp, $lr = LDRXpost $sp, 16
+    ;
+    ; CHECK-ASYNC-LABEL: name:            stack-save
+    ; CHECK-ASYNC-NOT: BL
     $w11 = ORRWri $wzr, 1
     $w11 = ORRWri $wzr, 1
     $w11 = ORRWri $wzr, 1
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-drop-stack.mir b/llvm/test/CodeGen/AArch64/machine-outliner-drop-stack.mir
index b39c5f5c6aa18a..4db0c00c608e84 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-drop-stack.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-drop-stack.mir
@@ -1,12 +1,15 @@
-# RUN: llc -mtriple=aarch64--- -run-pass=machine-outliner \
-# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64 -run-pass=machine-outliner -verify-machineinstrs %s -o - | \
+# RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-SYNC
+# RUN: llc -mtriple=aarch64 -run-pass=machine-outliner -verify-machineinstrs \
+# RUN:     -aarch64-enable-async-unwind-for-minsize %s -o - | \
+# RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-ASYNC
 
 --- |
   define void @no-save1() #0 { ret void }
   define void @no-save2() #0 { ret void }
   define void @reg-save() #0 { ret void }
   define void @stack-save() #0 { ret void }
-  attributes #0 = { minsize noinline noredzone "frame-pointer"="all" }
+  attributes #0 = { minsize noinline noredzone "frame-pointer"="all" uwtable }
 ...
 ---
 
@@ -63,10 +66,17 @@ body:             |
   $lr = ORRXri $xzr, 1
   bb.1:
   liveins: $lr
-    ; CHECK-LABEL: name:            reg-save
-    ; CHECK: $[[REG:x[0-9]+]] = ORRXrs $xzr, $lr, 0
-    ; CHECK-NEXT: BL [[FN]]
-    ; CHECK-NEXT: $lr = ORRXrs $xzr, $[[REG]], 0
+    ; CHECK-SYNC-LABEL: name:            reg-save
+    ; CHECK-SYNC: $x[[REG:[0-9]+]] = ORRXrs $xzr, $lr, 0
+    ; CHECK-SYNC-NEXT: BL [[FN]]
+    ; CHECK-SYNC-NEXT: $lr = ORRXrs $xzr, $x[[REG]], 0
+    ;
+    ; CHECK-ASYNC-LABEL: name:            reg-save
+    ; CHECK-ASYNC: $x[[REG:[0-9]+]] = ORRXrs $xzr, $lr, 0
+    ; CHECK-ASYNC-NEXT: CFI_INSTRUCTION register $w30, $w[[REG]]
+    ; CHECK-ASYNC-NEXT: BL [[FN]]
+    ; CHECK-ASYNC-NEXT: $lr = ORRXrs $xzr, $x[[REG]], 0
+    ; CHECK-ASYNC-NEXT: CFI_INSTRUCTION restore $w30
     $x12 = ADDXri $sp, 48, 0;
     $x12 = ADDXri $sp, 48, 0;
     $x12 = ADDXri $sp, 48, 0;
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-regsave.mir b/llvm/test/CodeGen/AArch64/machine-outliner-regsave.mir
index 91b8ecfdb9bdc1..0979cd6b2391c0 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-regsave.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-regsave.mir
@@ -1,5 +1,10 @@
 # RUN: llc -mtriple=aarch64-apple-darwin -run-pass=prolog-epilog \
-# RUN: -run-pass=machine-outliner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN:     -run-pass=machine-outliner -verify-machineinstrs %s -o - | \
+# RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-SYNC
+# RUN: llc -mtriple=aarch64-apple-darwin -run-pass=prolog-epilog \
+# RUN:     -run-pass=machine-outliner -verify-machineinstrs \
+# RUN:     -aarch64-enable-async-unwind-for-minsize %s -o - | \
+# RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-ASYNC
 # Check that we save LR to a callee-saved register when possible.
 # foo() should use a callee-saved register. However, bar() should not.
 --- |
@@ -12,7 +17,7 @@
     ret void
   }
 
-  attributes #0 = { minsize noinline noredzone "frame-pointer"="all" }
+  attributes #0 = { minsize noinline noredzone "frame-pointer"="all" uwtable }
 ...
 ---
 # Make sure that when we outline and a register is available, we
@@ -22,16 +27,22 @@
 # CHECK: name:            foo
 # CHECK-DAG: bb.1
 # CHECK-DAG: $x[[REG:[0-9]+]] = ORRXrs $xzr, $lr, 0
+# CHECK-ASYNC-NEXT: CFI_INSTRUCTION register $w30, $w[[REG]]
 # CHECK-NEXT: BL
 # CHECK-NEXT: $lr = ORRXrs $xzr, $x[[REG]], 0
+# CHECK-ASYNC-NEXT: CFI_INSTRUCTION restore $w30
 # CHECK-DAG: bb.2
 # CHECK-DAG: $x[[REG]] = ORRXrs $xzr, $lr, 0
+# CHECK-ASYNC-NEXT: CFI_INSTRUCTION register $w30, $w[[REG]]
 # CHECK-NEXT: BL
 # CHECK-NEXT: $lr = ORRXrs $xzr, $x[[REG]], 0
+# CHECK-ASYNC-NEXT: CFI_INSTRUCTION restore $w30
 # CHECK-DAG: bb.3
 # CHECK-DAG: $x[[REG]] = ORRXrs $xzr, $lr, 0
+# CHECK-ASYNC-NEXT: CFI_INSTRUCTION register $w30, $w[[REG]]
 # CHECK-NEXT: BL
 # CHECK-NEXT: $lr = ORRXrs $xzr, $x[[REG]], 0
+# CHECK-ASYNC-NEXT: CFI_INSTRUCTION restore $w30
 # CHECK-DAG: bb.4
 # CHECK-NOT: $x[[REG]] = ORRXrs $xzr, $lr, 0
 # CHECK-DAG: BL
@@ -86,16 +97,19 @@ body:             |
 # nothing to be outlined.
 # They also deliberately don't contain x18 to show that on Darwin we won't store
 # to that.
-# CHECK-LABEL: name:            bar
-# CHECK: early-clobber $sp = STRXpre $lr, $sp, -16
-# CHECK-NEXT: BL
-# CHECK-DAG: early-clobber $sp, $lr = LDRXpost $sp, 16
-# CHECK: early-clobber $sp = STRXpre $lr, $sp, -16
-# CHECK-NEXT: BL
-# CHECK-DAG: early-clobber $sp, $lr = LDRXpost $sp, 16
-# CHECK: early-clobber $sp = STRXpre $lr, $sp, -16
-# CHECK-NEXT: BL
-# CHECK-NEXT: early-clobber $sp, $lr = LDRXpost $sp, 16
+# CHECK-SYNC-LABEL: name:            bar
+# CHECK-SYNC: early-clobber $sp = STRXpre $lr, $sp, -16
+# CHECK-SYNC-NEXT: BL
+# CHECK-SYNC-DAG: early-clobber $sp, $lr = LDRXpost $sp, 16
+# CHECK-SYNC: early-clobber $sp = STRXpre $lr, $sp, -16
+# CHECK-SYNC-NEXT: BL
+# CHECK-SYNC-DAG: early-clobber $sp, $lr = LDRXpost $sp, 16
+# CHECK-SYNC: early-clobber $sp = STRXpre $lr, $sp, -16
+# CHECK-SYNC-NEXT: BL
+# CHECK-SYNC-NEXT: early-clobber $sp, $lr = LDRXpost $sp, 16
+#
+# CHECK-ASYNC-LABEL: name:            bar
+# CHECK-ASYNC-NOT: BL
 name:            bar
 tracksRegLiveness: true
 body:             |



More information about the llvm-branch-commits mailing list