[clang] [lld] [llvm] [Clang][CGData] Flags (PR #90304)
Kyungwoo Lee via llvm-commits
llvm-commits at lists.llvm.org
Thu May 2 19:31:39 PDT 2024
https://github.com/kyulee-com updated https://github.com/llvm/llvm-project/pull/90304
>From dd8a1fd0125a0bad48bd83492cfe7f6c1fc4c1e0 Mon Sep 17 00:00:00 2001
From: Xuan Zhang <xuanzh at meta.com>
Date: Fri, 26 Apr 2024 12:44:02 -0700
Subject: [PATCH 1/9] efficient implementation of
MachineOutliner::findCandidates()
---
llvm/lib/CodeGen/MachineOutliner.cpp | 22 ++++++++++---------
.../AArch64/machine-outliner-overlap.mir | 12 +++++-----
2 files changed, 18 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index dc2f5ef15206e8..d553c0e6d24772 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -593,7 +593,11 @@ void MachineOutliner::findCandidates(
unsigned NumDiscarded = 0;
unsigned NumKept = 0;
#endif
- for (const unsigned &StartIdx : RS.StartIndices) {
+ // Sort the start indices so that we can efficiently check if candidates
+ // overlap with each other in MachineOutliner::findCandidates().
+ SmallVector<unsigned> SortedStartIndices(RS.StartIndices);
+ llvm::sort(SortedStartIndices);
+ for (const unsigned &StartIdx : SortedStartIndices) {
// Trick: Discard some candidates that would be incompatible with the
// ones we've already found for this sequence. This will save us some
// work in candidate selection.
@@ -616,17 +620,15 @@ void MachineOutliner::findCandidates(
// * End before the other starts
// * Start after the other ends
unsigned EndIdx = StartIdx + StringLen - 1;
- auto FirstOverlap = find_if(
- CandidatesForRepeatedSeq, [StartIdx, EndIdx](const Candidate &C) {
- return EndIdx >= C.getStartIdx() && StartIdx <= C.getEndIdx();
- });
- if (FirstOverlap != CandidatesForRepeatedSeq.end()) {
+ if (CandidatesForRepeatedSeq.size() > 0 &&
+ StartIdx <= CandidatesForRepeatedSeq.back().getEndIdx()) {
#ifndef NDEBUG
++NumDiscarded;
- LLVM_DEBUG(dbgs() << " .. DISCARD candidate @ [" << StartIdx
- << ", " << EndIdx << "]; overlaps with candidate @ ["
- << FirstOverlap->getStartIdx() << ", "
- << FirstOverlap->getEndIdx() << "]\n");
+ LLVM_DEBUG(dbgs() << " .. DISCARD candidate @ [" << StartIdx << ", "
+ << EndIdx << "]; overlaps with candidate @ ["
+ << CandidatesForRepeatedSeq.back().getStartIdx()
+ << ", " << CandidatesForRepeatedSeq.back().getEndIdx()
+ << "]\n");
#endif
continue;
}
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir b/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir
index 649bb33828c32c..c6bd4c1d04d871 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir
@@ -8,27 +8,27 @@
# CHECK-NEXT: Candidates discarded: 0
# CHECK-NEXT: Candidates kept: 2
# CHECK-DAG: Sequence length: 8
-# CHECK-NEXT: .. DISCARD candidate @ [5, 12]; overlaps with candidate @ [12, 19]
+# CHECK-NEXT: .. DISCARD candidate @ [12, 19]; overlaps with candidate @ [5, 12]
# CHECK-NEXT: Candidates discarded: 1
# CHECK-NEXT: Candidates kept: 1
# CHECK-DAG: Sequence length: 9
-# CHECK-NEXT: .. DISCARD candidate @ [4, 12]; overlaps with candidate @ [11, 19]
+# CHECK-NEXT: .. DISCARD candidate @ [11, 19]; overlaps with candidate @ [4, 12]
# CHECK-NEXT: Candidates discarded: 1
# CHECK-NEXT: Candidates kept: 1
# CHECK-DAG: Sequence length: 10
-# CHECK-NEXT: .. DISCARD candidate @ [3, 12]; overlaps with candidate @ [10, 19]
+# CHECK-NEXT: .. DISCARD candidate @ [10, 19]; overlaps with candidate @ [3, 12]
# CHECK-NEXT: Candidates discarded: 1
# CHECK-NEXT: Candidates kept: 1
# CHECK-DAG: Sequence length: 11
-# CHECK-NEXT: .. DISCARD candidate @ [2, 12]; overlaps with candidate @ [9, 19]
+# CHECK-NEXT: .. DISCARD candidate @ [9, 19]; overlaps with candidate @ [2, 12]
# CHECK-NEXT: Candidates discarded: 1
# CHECK-NEXT: Candidates kept: 1
# CHECK-DAG: Sequence length: 12
-# CHECK-NEXT: .. DISCARD candidate @ [1, 12]; overlaps with candidate @ [8, 19]
+# CHECK-NEXT: .. DISCARD candidate @ [8, 19]; overlaps with candidate @ [1, 12]
# CHECK-NEXT: Candidates discarded: 1
# CHECK-NEXT: Candidates kept: 1
# CHECK-DAG: Sequence length: 13
-# CHECK-NEXT: .. DISCARD candidate @ [0, 12]; overlaps with candidate @ [7, 19]
+# CHECK-NEXT: .. DISCARD candidate @ [7, 19]; overlaps with candidate @ [0, 12]
# CHECK-NEXT: Candidates discarded: 1
# CHECK-NEXT: Candidates kept: 1
>From 5b8fb073f50490bd87d9be8eb36e5d7d0a9b9c96 Mon Sep 17 00:00:00 2001
From: Xuan Zhang <xuanzh at meta.com>
Date: Fri, 26 Apr 2024 13:04:38 -0700
Subject: [PATCH 2/9] outlining order by benefit-to-cost ratio
---
llvm/lib/CodeGen/MachineOutliner.cpp | 6 +-
.../machine-outliner-sort-per-priority.ll | 96 +++++++++++++++++++
.../CodeGen/ARM/machine-outliner-calls.mir | 80 ++++++----------
.../CodeGen/ARM/machine-outliner-default.mir | 33 +++----
.../ARM/machine-outliner-stack-fixup-arm.mir | 56 +++++------
.../machine-outliner-stack-fixup-thumb.mir | 74 ++++++--------
6 files changed, 194 insertions(+), 151 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index d553c0e6d24772..bc21215a181f6c 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -830,10 +830,12 @@ bool MachineOutliner::outline(Module &M,
<< "\n");
bool OutlinedSomething = false;
- // Sort by benefit. The most beneficial functions should be outlined first.
+ // Sort by priority where priority := getNotOutlinedCost / getOutliningCost.
+ // The function with highest priority should be outlined first.
stable_sort(FunctionList,
[](const OutlinedFunction &LHS, const OutlinedFunction &RHS) {
- return LHS.getBenefit() > RHS.getBenefit();
+ return LHS.getNotOutlinedCost() * RHS.getOutliningCost() >
+ RHS.getNotOutlinedCost() * LHS.getOutliningCost();
});
// Walk over each function, outlining them as we go along. Functions are
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll
new file mode 100644
index 00000000000000..00efc3c6e71c89
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-sort-per-priority.ll
@@ -0,0 +1,96 @@
+; This tests the order in which functions are outlined in MachineOutliner
+; There are TWO key OutlinedFunction in FunctionList
+;
+; ===================== First One =====================
+; ```
+; mov w0, #1
+; mov w1, #2
+; mov w2, #3
+; mov w3, #4
+; mov w4, #5
+; ```
+; It has:
+; - `SequenceSize=20` and `OccurrenceCount=6`
+; - each Candidate has `CallOverhead=12` and `FrameOverhead=4`
+; - `NotOutlinedCost=20*6=120` and `OutliningCost=12*6+20+4=96`
+; - `Benefit=120-96=24` and `Priority=120/96=1.25`
+;
+; ===================== Second One =====================
+; ```
+; mov w6, #6
+; mov w7, #7
+; b
+; ```
+; It has:
+; - `SequenceSize=12` and `OccurrenceCount=4`
+; - each Candidate has `CallOverhead=4` and `FrameOverhead=0`
+; - `NotOutlinedCost=12*4=48` and `OutliningCost=4*4+12+0=28`
+; - `Benefit=120-96=20` and `Priority=48/28=1.71`
+;
+; Note that the first one has higher benefit, but lower priority.
+; Hence, when outlining per priority, the second one will be outlined first.
+
+; RUN: llc %s -enable-machine-outliner=always -filetype=obj -o %t
+; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-SORT-BY-PRIORITY
+
+; RUN: llc %s -enable-machine-outliner=always -outliner-benefit-threshold=22 -filetype=obj -o %t
+; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-THRESHOLD
+
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-macosx14.0.0"
+
+declare i32 @_Z3fooiiii(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef)
+
+define i32 @_Z2f1v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 11, i32 noundef 6, i32 noundef 7)
+ ret i32 %1
+}
+
+define i32 @_Z2f2v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 12, i32 noundef 6, i32 noundef 7)
+ ret i32 %1
+}
+
+define i32 @_Z2f3v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 13, i32 noundef 6, i32 noundef 7)
+ ret i32 %1
+}
+
+define i32 @_Z2f4v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 14, i32 noundef 6, i32 noundef 7)
+ ret i32 %1
+}
+
+define i32 @_Z2f5v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 15, i32 noundef 8, i32 noundef 9)
+ ret i32 %1
+}
+
+define i32 @_Z2f6v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 16, i32 noundef 9, i32 noundef 8)
+ ret i32 %1
+}
+
+; CHECK-SORT-BY-PRIORITY: <_OUTLINED_FUNCTION_0>:
+; CHECK-SORT-BY-PRIORITY-NEXT: mov w6, #0x6
+; CHECK-SORT-BY-PRIORITY-NEXT: mov w7, #0x7
+; CHECK-SORT-BY-PRIORITY-NEXT: b
+
+; CHECK-SORT-BY-PRIORITY: <_OUTLINED_FUNCTION_1>:
+; CHECK-SORT-BY-PRIORITY-NEXT: mov w0, #0x1
+; CHECK-SORT-BY-PRIORITY-NEXT: mov w1, #0x2
+; CHECK-SORT-BY-PRIORITY-NEXT: mov w2, #0x3
+; CHECK-SORT-BY-PRIORITY-NEXT: mov w3, #0x4
+; CHECK-SORT-BY-PRIORITY-NEXT: mov w4, #0x5
+; CHECK-SORT-BY-PRIORITY-NEXT: ret
+
+; CHECK-THRESHOLD: <_OUTLINED_FUNCTION_0>:
+; CHECK-THRESHOLD-NEXT: mov w0, #0x1
+; CHECK-THRESHOLD-NEXT: mov w1, #0x2
+; CHECK-THRESHOLD-NEXT: mov w2, #0x3
+; CHECK-THRESHOLD-NEXT: mov w3, #0x4
+; CHECK-THRESHOLD-NEXT: mov w4, #0x5
+; CHECK-THRESHOLD-NEXT: ret
+
+; CHECK-THRESHOLD-NOT: <_OUTLINED_FUNCTION_1>:
diff --git a/llvm/test/CodeGen/ARM/machine-outliner-calls.mir b/llvm/test/CodeGen/ARM/machine-outliner-calls.mir
index a92c9dd28be5ae..7634ecd6e863ae 100644
--- a/llvm/test/CodeGen/ARM/machine-outliner-calls.mir
+++ b/llvm/test/CodeGen/ARM/machine-outliner-calls.mir
@@ -26,15 +26,15 @@ body: |
; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8
; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4
; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -8
- ; CHECK: BL @OUTLINED_FUNCTION_0
+ ; CHECK: BL @OUTLINED_FUNCTION_2
; CHECK: bb.1:
- ; CHECK: BL @OUTLINED_FUNCTION_0
+ ; CHECK: BL @OUTLINED_FUNCTION_2
; CHECK: bb.2:
- ; CHECK: BL @OUTLINED_FUNCTION_0
+ ; CHECK: BL @OUTLINED_FUNCTION_2
; CHECK: bb.3:
- ; CHECK: BL @OUTLINED_FUNCTION_0
+ ; CHECK: BL @OUTLINED_FUNCTION_2
; CHECK: bb.4:
- ; CHECK: BL @OUTLINED_FUNCTION_0
+ ; CHECK: BL @OUTLINED_FUNCTION_2
; CHECK: bb.5:
; CHECK: $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr
; CHECK: BX_RET 14 /* CC::al */, $noreg
@@ -139,13 +139,13 @@ body: |
; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8
; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4
; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -8
- ; CHECK: BL @OUTLINED_FUNCTION_1
+ ; CHECK: BL @OUTLINED_FUNCTION_0
; CHECK: bb.1:
- ; CHECK: BL @OUTLINED_FUNCTION_1
+ ; CHECK: BL @OUTLINED_FUNCTION_0
; CHECK: bb.2:
- ; CHECK: BL @OUTLINED_FUNCTION_1
+ ; CHECK: BL @OUTLINED_FUNCTION_0
; CHECK: bb.3:
- ; CHECK: BL @OUTLINED_FUNCTION_1
+ ; CHECK: BL @OUTLINED_FUNCTION_0
; CHECK: bb.4:
; CHECK: $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr
; CHECK: BX_RET 14 /* CC::al */, $noreg
@@ -245,19 +245,19 @@ body: |
; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4
; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -8
; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp
- ; CHECK: BL @OUTLINED_FUNCTION_2
+ ; CHECK: BL @OUTLINED_FUNCTION_1
; CHECK: bb.1:
; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp
- ; CHECK: BL @OUTLINED_FUNCTION_2
+ ; CHECK: BL @OUTLINED_FUNCTION_1
; CHECK: bb.2:
; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp
- ; CHECK: BL @OUTLINED_FUNCTION_2
+ ; CHECK: BL @OUTLINED_FUNCTION_1
; CHECK: bb.3:
; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp
- ; CHECK: BL @OUTLINED_FUNCTION_2
+ ; CHECK: BL @OUTLINED_FUNCTION_1
; CHECK: bb.4:
; CHECK: BL @"\01mcount", csr_aapcs, implicit-def dead $lr, implicit $sp
- ; CHECK: BL @OUTLINED_FUNCTION_2
+ ; CHECK: BL @OUTLINED_FUNCTION_1
; CHECK: bb.5:
; CHECK: $sp = frame-destroy LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r4, def $lr
; CHECK: BX_RET 14 /* CC::al */, $noreg
@@ -307,38 +307,17 @@ body: |
bb.0:
BX_RET 14, $noreg
-
; CHECK-LABEL: name: OUTLINED_FUNCTION_0
; CHECK: bb.0:
- ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr
- ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8
- ; CHECK: BL @bar, implicit-def dead $lr, implicit $sp
- ; CHECK: $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $r2 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $r3 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
- ; CHECK: MOVPCLR 14 /* CC::al */, $noreg
-
- ; CHECK-LABEL: name: OUTLINED_FUNCTION_1
- ; CHECK: bb.0:
- ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr
- ; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8
- ; CHECK: BL @bar, implicit-def dead $lr, implicit $sp
+ ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
; CHECK: $r0 = MOVi 2, 14 /* CC::al */, $noreg, $noreg
; CHECK: $r1 = MOVi 2, 14 /* CC::al */, $noreg, $noreg
; CHECK: $r2 = MOVi 2, 14 /* CC::al */, $noreg, $noreg
; CHECK: $r3 = MOVi 2, 14 /* CC::al */, $noreg, $noreg
; CHECK: $r4 = MOVi 2, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
; CHECK: TAILJMPd @bar, implicit $sp
- ; CHECK-LABEL: name: OUTLINED_FUNCTION_2
+ ; CHECK-LABEL: name: OUTLINED_FUNCTION_1
; CHECK: bb.0:
; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
; CHECK: $r0 = MOVi 3, 14 /* CC::al */, $noreg, $noreg
@@ -348,31 +327,28 @@ body: |
; CHECK: $r4 = MOVi 3, 14 /* CC::al */, $noreg, $noreg
; CHECK: MOVPCLR 14 /* CC::al */, $noreg
+ ; CHECK-LABEL: name: OUTLINED_FUNCTION_2
+ ; CHECK: bb.0:
+ ; CHECK: liveins: $r11, $r10, $r9, $r8, $r7, $r6, $r5, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
+ ; CHECK: $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK: $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK: $r2 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK: $r3 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK: $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK: MOVPCLR 14 /* CC::al */, $noreg
+
; CHECK-LABEL: name: OUTLINED_FUNCTION_3
; CHECK: bb.0:
- ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr
- ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8
- ; CHECK: tBL 14 /* CC::al */, $noreg, @bar, implicit-def dead $lr, implicit $sp
+ ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
; CHECK: $r0 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg
; CHECK: $r1 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg
; CHECK: $r2 = t2MOVi 2, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
; CHECK: tTAILJMPdND @bar, 14 /* CC::al */, $noreg, implicit $sp
; CHECK-LABEL: name: OUTLINED_FUNCTION_4
; CHECK: bb.0:
- ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8, $lr
- ; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -8
- ; CHECK: tBL 14 /* CC::al */, $noreg, @bar, implicit-def dead $lr, implicit $sp
+ ; CHECK: liveins: $r11, $r10, $r9, $r8, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
; CHECK: $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
; CHECK: $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
; CHECK: $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
; CHECK: tBX_RET 14 /* CC::al */, $noreg
-
-
-
diff --git a/llvm/test/CodeGen/ARM/machine-outliner-default.mir b/llvm/test/CodeGen/ARM/machine-outliner-default.mir
index 6d0218dbfe636d..de2b8f55969765 100644
--- a/llvm/test/CodeGen/ARM/machine-outliner-default.mir
+++ b/llvm/test/CodeGen/ARM/machine-outliner-default.mir
@@ -19,17 +19,17 @@ body: |
; CHECK: bb.0:
; CHECK: liveins: $lr
; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ; CHECK: BL @OUTLINED_FUNCTION_0
+ ; CHECK: BL @OUTLINED_FUNCTION_1
; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
; CHECK: bb.1:
; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11
; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ; CHECK: BL @OUTLINED_FUNCTION_0
+ ; CHECK: BL @OUTLINED_FUNCTION_1
; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
; CHECK: bb.2:
; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11
; CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ; CHECK: BL @OUTLINED_FUNCTION_0
+ ; CHECK: BL @OUTLINED_FUNCTION_1
; CHECK: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
; CHECK: bb.3:
; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11
@@ -73,17 +73,17 @@ body: |
; CHECK: bb.0:
; CHECK: liveins: $lr
; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1
+ ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0
; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
; CHECK: bb.1:
; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11
; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1
+ ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0
; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
; CHECK: bb.2:
; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11
; CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_1
+ ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0
; CHECK: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
; CHECK: bb.3:
; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11
@@ -114,6 +114,15 @@ body: |
; CHECK-LABEL: name: OUTLINED_FUNCTION_0
; CHECK: bb.0:
+ ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11
+ ; CHECK: $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK: $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK: $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK: $r3 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK: tBX_RET 14 /* CC::al */, $noreg
+
+ ; CHECK-LABEL: name: OUTLINED_FUNCTION_1
+ ; CHECK: bb.0:
; CHECK: liveins: $lr, $r6, $r7, $r8, $r9, $r10, $r11
; CHECK: $r0 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
; CHECK: $r1 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
@@ -122,15 +131,3 @@ body: |
; CHECK: $r4 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
; CHECK: $r5 = MOVi 1, 14 /* CC::al */, $noreg, $noreg
; CHECK: MOVPCLR 14 /* CC::al */, $noreg
-
- ; CHECK-LABEL: name: OUTLINED_FUNCTION_1
- ; CHECK: bb.0:
- ; CHECK: liveins: $lr, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11
- ; CHECK: $r0 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $r1 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $r2 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: $r3 = t2MOVi 1, 14 /* CC::al */, $noreg, $noreg
- ; CHECK: tBX_RET 14 /* CC::al */, $noreg
-
-
-
diff --git a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir
index ae5caa5b7c06db..e71edc8ceb3f60 100644
--- a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir
+++ b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-arm.mir
@@ -18,6 +18,7 @@ body: |
liveins: $r0
; CHECK-LABEL: name: CheckAddrMode_i12
; CHECK: $r1 = MOVr killed $r0, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I12:[0-9]+]]
; CHECK-NEXT: $r6 = LDRi12 $sp, 4088, 14 /* CC::al */, $noreg
$r1 = MOVr killed $r0, 14, $noreg, $noreg
@@ -47,6 +48,7 @@ body: |
liveins: $r1
; CHECK-LABEL: name: CheckAddrMode3
; CHECK: $r0 = MOVr killed $r1, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I3:[0-9]+]]
; CHECK-NEXT: $r6 = LDRSH $sp, $noreg, 248, 14 /* CC::al */, $noreg
$r0 = MOVr killed $r1, 14, $noreg, $noreg
@@ -76,6 +78,7 @@ body: |
liveins: $r2
; CHECK-LABEL: name: CheckAddrMode5
; CHECK: $r0 = MOVr killed $r2, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I5:[0-9]+]]
; CHECK-NEXT: $d5 = VLDRD $sp, 254, 14 /* CC::al */, $noreg
$r0 = MOVr killed $r2, 14, $noreg, $noreg
@@ -110,6 +113,7 @@ body: |
liveins: $r3
; CHECK-LABEL: name: CheckAddrMode5FP16
; CHECK: $r0 = MOVr killed $r3, 14 /* CC::al */, $noreg, $noreg
+ ; CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
; CHECK-NEXT: BL @OUTLINED_FUNCTION_[[I5FP16:[0-9]+]]
; CHECK-NEXT: $s6 = VLDRH $sp, 252, 14, $noreg
$r0 = MOVr killed $r3, 14, $noreg, $noreg
@@ -146,41 +150,29 @@ body: |
BX_RET 14, $noreg
;CHECK: name: OUTLINED_FUNCTION_[[I5]]
- ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
- ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
- ;CHECK-NEXT: $d0 = VLDRD $sp, 2, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $d1 = VLDRD $sp, 10, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $d4 = VLDRD $sp, 255, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
+ ;CHECK: liveins: $r10, $r9, $r8, $r7, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
+ ;CHECK: $d0 = VLDRD $sp, 0, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: $d1 = VLDRD $sp, 8, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: $d4 = VLDRD $sp, 253, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg
;CHECK: name: OUTLINED_FUNCTION_[[I5FP16]]
- ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
- ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
- ;CHECK-NEXT: $s1 = VLDRH $sp, 4, 14, $noreg
- ;CHECK-NEXT: $s2 = VLDRH $sp, 12, 14, $noreg
- ;CHECK-NEXT: $s5 = VLDRH $sp, 244, 14, $noreg
- ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
+ ;CHECK: liveins: $r10, $r9, $r8, $r7, $r6, $r5, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9, $d8
+ ;CHECK: $s1 = VLDRH $sp, 0, 14, $noreg
+ ;CHECK-NEXT: $s2 = VLDRH $sp, 8, 14, $noreg
+ ;CHECK-NEXT: $s5 = VLDRH $sp, 240, 14, $noreg
+ ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg
;CHECK: name: OUTLINED_FUNCTION_[[I12]]
- ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
- ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
- ;CHECK-NEXT: $r1 = LDRi12 $sp, 8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $r2 = LDRi12 $sp, 16, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $r5 = LDRi12 $sp, 4094, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
+ ;CHECK: liveins: $r10, $r9, $r8, $r7, $d8, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9
+ ;CHECK: $r1 = LDRi12 $sp, 0, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: $r2 = LDRi12 $sp, 8, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: $r5 = LDRi12 $sp, 4086, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg
;CHECK: name: OUTLINED_FUNCTION_[[I3]]
- ;CHECK: early-clobber $sp = frame-setup STR_PRE_IMM killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
- ;CHECK-NEXT: BL @foo, implicit-def dead $lr, implicit $sp
- ;CHECK-NEXT: $r1 = LDRSH $sp, $noreg, 8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $r2 = LDRSH $sp, $noreg, 16, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $r5 = LDRSH $sp, $noreg, 255, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $lr, $sp = frame-destroy LDR_POST_IMM $sp, $noreg, 8, 14 /* CC::al */, $noreg
+ ;CHECK: liveins: $r10, $r9, $r8, $r7, $d8, $r4, $d15, $d14, $d13, $d12, $d11, $d10, $d9
+ ;CHECK: $r1 = LDRSH $sp, $noreg, 0, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: $r2 = LDRSH $sp, $noreg, 8, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: $r5 = LDRSH $sp, $noreg, 247, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: BX_RET 14 /* CC::al */, $noreg
diff --git a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir
index 56184448424489..f2cdaebdfa8baa 100644
--- a/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir
+++ b/llvm/test/CodeGen/ARM/machine-outliner-stack-fixup-thumb.mir
@@ -19,7 +19,7 @@ body: |
bb.0:
liveins: $r1
;CHECK-LABEL: name: CheckAddrModeT2_i12
- ;CHECK: $r0 = tMOVr killed $r1, 14 /* CC::al */, $noreg
+ ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]]
;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I12:[0-9]+]]
;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4088, 14 /* CC::al */, $noreg
$r0 = tMOVr killed $r1, 14, $noreg
@@ -49,7 +49,7 @@ body: |
bb.0:
liveins: $r1
;CHECK-LABEL: name: CheckAddrModeT2_i8
- ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg
+ ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]]
;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I8:[0-9]+]]
;CHECK-NEXT: t2STRHT $r0, $sp, 248, 14 /* CC::al */, $noreg
$r0 = tMOVr $r1, 14, $noreg
@@ -79,7 +79,7 @@ body: |
bb.0:
liveins: $r1
;CHECK-LABEL: name: CheckAddrModeT2_i8s4
- ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg
+ ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]]
;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[I8S4:[0-9]+]]
;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1020, 14 /* CC::al */, $noreg
$r0 = tMOVr $r1, 14, $noreg
@@ -109,7 +109,7 @@ body: |
bb.0:
liveins: $r1
;CHECK-LABEL: name: CheckAddrModeT2_ldrex
- ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg
+ ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]]
;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[LDREX:[0-9]+]]
;CHECK-NEXT: $r1 = t2LDREX $sp, 254, 14 /* CC::al */, $noreg
$r0 = tMOVr $r1, 14, $noreg
@@ -144,8 +144,10 @@ body: |
bb.0:
liveins: $r0, $r1
;CHECK-LABEL: name: CheckAddrModeT1_s
- ;CHECK: $r0 = tMOVr $r1, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[T1_S:[0-9]+]]
+ ;CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_[[SHARED:[0-9+]]]
+ ;CHECK-NEXT: tSTRspi $r0, $sp, 0, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: tSTRspi $r0, $sp, 4, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: tSTRspi $r0, $sp, 253, 14 /* CC::al */, $noreg
;CHECK-NEXT: tSTRspi $r0, $sp, 254, 14 /* CC::al */, $noreg
$r0 = tMOVr $r1, 14, $noreg
tBL 14, $noreg, @foo, implicit-def dead $lr, implicit $sp
@@ -181,51 +183,29 @@ body: |
BX_RET 14, $noreg
;CHECK: name: OUTLINED_FUNCTION_[[LDREX]]
- ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
- ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp
- ;CHECK-NEXT: $r1 = t2LDREX $sp, 2, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $r1 = t2LDREX $sp, 10, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $r1 = t2LDREX $sp, 255, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
+ ;CHECK: $r1 = t2LDREX $sp, 0, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: $r1 = t2LDREX $sp, 8, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: $r1 = t2LDREX $sp, 253, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg
;CHECK: name: OUTLINED_FUNCTION_[[I8]]
- ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
- ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp
- ;CHECK-NEXT: t2STRHT $r0, $sp, 8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: t2STRHT $r0, $sp, 12, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: t2STRHT $r0, $sp, 255, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
+ ;CHECK: t2STRHT $r0, $sp, 0, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: t2STRHT $r0, $sp, 4, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: t2STRHT $r0, $sp, 247, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg
;CHECK: name: OUTLINED_FUNCTION_[[I8S4]]
- ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
- ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp
+ ;CHECK: t2STRDi8 $r0, $r1, $sp, 0, 14 /* CC::al */, $noreg
;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 16, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1020, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: t2STRDi8 $r0, $r1, $sp, 1012, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $nore
;CHECK: name: OUTLINED_FUNCTION_[[I12]]
- ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
- ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp
- ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 12, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4094, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
-
- ;CHECK: name: OUTLINED_FUNCTION_[[T1_S]]
- ;CHECK: early-clobber $sp = frame-setup t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION def_cfa_offset 8
- ;CHECK-NEXT: frame-setup CFI_INSTRUCTION offset $lr, -8
- ;CHECK-NEXT: tBL 14 /* CC::al */, $noreg, @foo, implicit-def dead $lr, implicit $sp
- ;CHECK-NEXT: tSTRspi $r0, $sp, 2, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: tSTRspi $r0, $sp, 6, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: tSTRspi $r0, $sp, 255, 14 /* CC::al */, $noreg
- ;CHECK-NEXT: $lr, $sp = frame-destroy t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
+ ;CHECK: $r0 = t2LDRi12 $sp, 0, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: $r0 = t2LDRi12 $sp, 4086, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: tBX_RET 14 /* CC::al */, $noreg
+
+ ;CHECK: name: OUTLINED_FUNCTION_[[SHARED]]
+ ;CHECK: $r0 = tMOVr killed $r1, 14 /* CC::al */, $noreg
+ ;CHECK-NEXT: tTAILJMPdND @foo, 14 /* CC::al */, $noreg, implicit $sp
>From 75c515f4456237ac5c3eba3c3186b94c9503443c Mon Sep 17 00:00:00 2001
From: Xuan Zhang <xuanzh at meta.com>
Date: Fri, 12 Apr 2024 10:57:51 -0700
Subject: [PATCH 3/9] consider leaf descendants to include more candidates for
outlining
---
llvm/include/llvm/Support/SuffixTree.h | 34 ++++-
llvm/include/llvm/Support/SuffixTreeNode.h | 25 +++-
llvm/lib/CodeGen/MachineOutliner.cpp | 8 +-
llvm/lib/Support/SuffixTree.cpp | 83 ++++++++++-
llvm/lib/Support/SuffixTreeNode.cpp | 5 +
.../machine-outliner-cfi-tail-some.mir | 2 +-
.../machine-outliner-leaf-descendants.ll | 124 ++++++++++++++++
.../AArch64/machine-outliner-overlap.mir | 24 +++-
.../machine-outliner-retaddr-sign-sp-mod.mir | 2 +-
.../machine-outliner-retaddr-sign-thunk.ll | 4 +-
.../AArch64/machine-outliner-throw2.ll | 4 +-
.../CodeGen/AArch64/machine-outliner-thunk.ll | 2 +-
.../test/CodeGen/AArch64/machine-outliner.mir | 2 +-
.../RISCV/machineoutliner-pcrel-lo.mir | 8 +-
llvm/unittests/Support/SuffixTreeTest.cpp | 134 ++++++++++++++++++
15 files changed, 441 insertions(+), 20 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/machine-outliner-leaf-descendants.ll
diff --git a/llvm/include/llvm/Support/SuffixTree.h b/llvm/include/llvm/Support/SuffixTree.h
index 4940fbbf308d8b..37b73666404300 100644
--- a/llvm/include/llvm/Support/SuffixTree.h
+++ b/llvm/include/llvm/Support/SuffixTree.h
@@ -42,6 +42,9 @@ class SuffixTree {
/// Each element is an integer representing an instruction in the module.
ArrayRef<unsigned> Str;
+ /// Whether to consider leaf descendants or only leaf children.
+ bool OutlinerLeafDescendants;
+
/// A repeated substring in the tree.
struct RepeatedSubstring {
/// The length of the string.
@@ -130,11 +133,27 @@ class SuffixTree {
/// this step.
unsigned extend(unsigned EndIdx, unsigned SuffixesToAdd);
+ /// This vector contains all leaf nodes of this suffix tree. These leaf nodes
+ /// are identified using post-order depth-first traversal, so that the order
+ /// of these leaf nodes in the vector matches the order of the leaves in the
+ /// tree from left to right if one were to draw the tree on paper.
+ std::vector<SuffixTreeLeafNode *> LeafNodes;
+
+ /// Perform a post-order depth-first traversal of the tree and perform two
+ /// tasks during the traversal. The first is to populate LeafNodes, adding
+ /// nodes in order of the traversal. The second is to keep track of the leaf
+ /// descendants of every internal node by assigning values to LeftLeafIndex
+ /// and RightLefIndex fields of SuffixTreeNode for all internal nodes.
+ void setLeafNodes();
+
public:
/// Construct a suffix tree from a sequence of unsigned integers.
///
/// \param Str The string to construct the suffix tree for.
- SuffixTree(const ArrayRef<unsigned> &Str);
+ /// \param OutlinerLeafDescendants Whether to consider leaf descendants or
+ /// only leaf children (used by Machine Outliner).
+ SuffixTree(const ArrayRef<unsigned> &Str,
+ bool OutlinerLeafDescendants = false);
/// Iterator for finding all repeated substrings in the suffix tree.
struct RepeatedSubstringIterator {
@@ -154,6 +173,12 @@ class SuffixTree {
/// instruction lengths.
const unsigned MinLength = 2;
+ /// Vector of leaf nodes of the suffix tree.
+ const std::vector<SuffixTreeLeafNode *> &LeafNodes;
+
+ /// Whether to consider leaf descendants or only leaf children.
+ bool OutlinerLeafDescendants = !LeafNodes.empty();
+
/// Move the iterator to the next repeated substring.
void advance();
@@ -179,7 +204,10 @@ class SuffixTree {
return !(*this == Other);
}
- RepeatedSubstringIterator(SuffixTreeInternalNode *N) : N(N) {
+ RepeatedSubstringIterator(
+ SuffixTreeInternalNode *N,
+ const std::vector<SuffixTreeLeafNode *> &LeafNodes = {})
+ : N(N), LeafNodes(LeafNodes) {
// Do we have a non-null node?
if (!N)
return;
@@ -191,7 +219,7 @@ class SuffixTree {
};
typedef RepeatedSubstringIterator iterator;
- iterator begin() { return iterator(Root); }
+ iterator begin() { return iterator(Root, LeafNodes); }
iterator end() { return iterator(nullptr); }
};
diff --git a/llvm/include/llvm/Support/SuffixTreeNode.h b/llvm/include/llvm/Support/SuffixTreeNode.h
index 7d0d1cf0c58b95..84b590f2deb0cd 100644
--- a/llvm/include/llvm/Support/SuffixTreeNode.h
+++ b/llvm/include/llvm/Support/SuffixTreeNode.h
@@ -46,6 +46,17 @@ struct SuffixTreeNode {
/// the root to this node.
unsigned ConcatLen = 0;
+ /// These two indices give a range of indices for its leaf descendants.
+ /// Imagine drawing a tree on paper and assigning a unique index to each leaf
+ /// node in monotonically increasing order from left to right. This way of
+ /// numbering the leaf nodes allows us to associate a continuous range of
+ /// indices with each internal node. For example, if a node has leaf
+ /// descendants with indices i, i+1, ..., j, then its LeftLeafIdx is i and
+ /// its RightLeafIdx is j. These indices are for LeafNodes in the SuffixTree
+ /// class, which is constructed using post-order depth-first traversal.
+ unsigned LeftLeafIdx = EmptyIdx;
+ unsigned RightLeafIdx = EmptyIdx;
+
public:
// LLVM RTTI boilerplate.
NodeKind getKind() const { return Kind; }
@@ -56,6 +67,18 @@ struct SuffixTreeNode {
/// \returns the end index of this node.
virtual unsigned getEndIdx() const = 0;
+ /// \return the index of this node's left most leaf node.
+ unsigned getLeftLeafIdx() const;
+
+ /// \return the index of this node's right most leaf node.
+ unsigned getRightLeafIdx() const;
+
+ /// Set the index of the left most leaf node of this node to \p Idx.
+ void setLeftLeafIdx(unsigned Idx);
+
+ /// Set the index of the right most leaf node of this node to \p Idx.
+ void setRightLeafIdx(unsigned Idx);
+
/// Advance this node's StartIdx by \p Inc.
void incrementStartIdx(unsigned Inc);
@@ -168,4 +191,4 @@ struct SuffixTreeLeafNode : SuffixTreeNode {
virtual ~SuffixTreeLeafNode() = default;
};
} // namespace llvm
-#endif // LLVM_SUPPORT_SUFFIXTREE_NODE_H
\ No newline at end of file
+#endif // LLVM_SUPPORT_SUFFIXTREE_NODE_H
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index bc21215a181f6c..01a9966adb097a 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -121,6 +121,12 @@ static cl::opt<unsigned> OutlinerBenefitThreshold(
cl::desc(
"The minimum size in bytes before an outlining candidate is accepted"));
+static cl::opt<bool> OutlinerLeafDescendants(
+ "outliner-leaf-descendants", cl::init(true), cl::Hidden,
+ cl::desc("Consider all leaf descendants of internal nodes of the suffix "
+ "tree as candidates for outlining (if false, only leaf children "
+ "are considered)"));
+
namespace {
/// Maps \p MachineInstrs to unsigned integers and stores the mappings.
@@ -576,7 +582,7 @@ void MachineOutliner::emitOutlinedFunctionRemark(OutlinedFunction &OF) {
void MachineOutliner::findCandidates(
InstructionMapper &Mapper, std::vector<OutlinedFunction> &FunctionList) {
FunctionList.clear();
- SuffixTree ST(Mapper.UnsignedVec);
+ SuffixTree ST(Mapper.UnsignedVec, OutlinerLeafDescendants);
// First, find all of the repeated substrings in the tree of minimum length
// 2.
diff --git a/llvm/lib/Support/SuffixTree.cpp b/llvm/lib/Support/SuffixTree.cpp
index eaa653078e0900..4bd5cdfe8791dd 100644
--- a/llvm/lib/Support/SuffixTree.cpp
+++ b/llvm/lib/Support/SuffixTree.cpp
@@ -11,9 +11,11 @@
//===----------------------------------------------------------------------===//
#include "llvm/Support/SuffixTree.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/SuffixTreeNode.h"
+#include <stack>
using namespace llvm;
@@ -26,7 +28,9 @@ static size_t numElementsInSubstring(const SuffixTreeNode *N) {
return N->getEndIdx() - N->getStartIdx() + 1;
}
-SuffixTree::SuffixTree(const ArrayRef<unsigned> &Str) : Str(Str) {
+SuffixTree::SuffixTree(const ArrayRef<unsigned> &Str,
+ bool OutlinerLeafDescendants)
+ : Str(Str), OutlinerLeafDescendants(OutlinerLeafDescendants) {
Root = insertRoot();
Active.Node = Root;
@@ -46,6 +50,11 @@ SuffixTree::SuffixTree(const ArrayRef<unsigned> &Str) : Str(Str) {
// Set the suffix indices of each leaf.
assert(Root && "Root node can't be nullptr!");
setSuffixIndices();
+
+ // Collect all leaf nodes of the suffix tree. And for each internal node,
+ // record the range of leaf nodes that are descendants of it.
+ if (OutlinerLeafDescendants)
+ setLeafNodes();
}
SuffixTreeNode *SuffixTree::insertLeaf(SuffixTreeInternalNode &Parent,
@@ -105,6 +114,68 @@ void SuffixTree::setSuffixIndices() {
}
}
+void SuffixTree::setLeafNodes() {
+ // A stack that keeps track of nodes to visit for post-order DFS traversal.
+ std::stack<SuffixTreeNode *> ToVisit;
+ ToVisit.push(Root);
+
+ // This keeps track of the index of the next leaf node to be added to
+ // the LeafNodes vector of the suffix tree.
+ unsigned LeafCounter = 0;
+
+ // This keeps track of nodes whose children have been added to the stack
+ // during the post-order depth-first traversal of the tree.
+ llvm::SmallPtrSet<SuffixTreeInternalNode *, 32> ChildrenAddedToStack;
+
+ // Traverse the tree in post-order.
+ while (!ToVisit.empty()) {
+ SuffixTreeNode *CurrNode = ToVisit.top();
+ ToVisit.pop();
+ if (auto *CurrInternalNode = dyn_cast<SuffixTreeInternalNode>(CurrNode)) {
+ // The current node is an internal node.
+ if (ChildrenAddedToStack.find(CurrInternalNode) !=
+ ChildrenAddedToStack.end()) {
+ // If the children of the current node has been added to the stack,
+ // then this is the second time we visit this node and at this point,
+ // all of its children have already been processed. Now, we can
+ // set its LeftLeafIdx and RightLeafIdx;
+ auto it = CurrInternalNode->Children.begin();
+ if (it != CurrInternalNode->Children.end()) {
+ // Get the first child to use its RightLeafIdx. The RightLeafIdx is
+ // used as the first child is the initial one added to the stack, so
+ // it's the last one to be processed. This implies that the leaf
+ // descendants of the first child are assigned the largest index
+ // numbers.
+ CurrNode->setRightLeafIdx(it->second->getRightLeafIdx());
+ // get the last child to use its LeftLeafIdx.
+ while (std::next(it) != CurrInternalNode->Children.end())
+ it = std::next(it);
+ CurrNode->setLeftLeafIdx(it->second->getLeftLeafIdx());
+ assert(CurrNode->getLeftLeafIdx() <= CurrNode->getRightLeafIdx() &&
+ "LeftLeafIdx should not be larger than RightLeafIdx");
+ }
+ } else {
+ // This is the first time we visit this node. This means that its
+ // children have not been added to the stack yet. Hence, we will add
+ // the current node back to the stack and add its children to the
+ // stack for processing.
+ ToVisit.push(CurrNode);
+ for (auto &ChildPair : CurrInternalNode->Children)
+ ToVisit.push(ChildPair.second);
+ ChildrenAddedToStack.insert(CurrInternalNode);
+ }
+ } else {
+ // The current node is a leaf node.
+ // We can simplyset its LeftLeafIdx and RightLeafIdx.
+ CurrNode->setLeftLeafIdx(LeafCounter);
+ CurrNode->setRightLeafIdx(LeafCounter);
+ LeafCounter++;
+ auto *CurrLeafNode = cast<SuffixTreeLeafNode>(CurrNode);
+ LeafNodes.push_back(CurrLeafNode);
+ }
+ }
+}
+
unsigned SuffixTree::extend(unsigned EndIdx, unsigned SuffixesToAdd) {
SuffixTreeInternalNode *NeedsLink = nullptr;
@@ -230,6 +301,7 @@ void SuffixTree::RepeatedSubstringIterator::advance() {
// Each leaf node represents a repeat of a string.
SmallVector<unsigned> RepeatedSubstringStarts;
+ SmallVector<SuffixTreeLeafNode *> LeafDescendants;
// Continue visiting nodes until we find one which repeats more than once.
while (!InternalNodesToVisit.empty()) {
@@ -252,7 +324,7 @@ void SuffixTree::RepeatedSubstringIterator::advance() {
continue;
}
- if (Length < MinLength)
+ if (Length < MinLength || OutlinerLeafDescendants)
continue;
// Have an occurrence of a potentially repeated string. Save it.
@@ -260,6 +332,13 @@ void SuffixTree::RepeatedSubstringIterator::advance() {
RepeatedSubstringStarts.push_back(Leaf->getSuffixIdx());
}
+ if (OutlinerLeafDescendants && Length >= MinLength) {
+ LeafDescendants.assign(LeafNodes.begin() + Curr->getLeftLeafIdx(),
+ LeafNodes.begin() + Curr->getRightLeafIdx() + 1);
+ for (SuffixTreeLeafNode *Leaf : LeafDescendants)
+ RepeatedSubstringStarts.push_back(Leaf->getSuffixIdx());
+ }
+
// The root never represents a repeated substring. If we're looking at
// that, then skip it.
if (Curr->isRoot())
diff --git a/llvm/lib/Support/SuffixTreeNode.cpp b/llvm/lib/Support/SuffixTreeNode.cpp
index 113b990fd352fc..9f1f94a39895e8 100644
--- a/llvm/lib/Support/SuffixTreeNode.cpp
+++ b/llvm/lib/Support/SuffixTreeNode.cpp
@@ -38,3 +38,8 @@ unsigned SuffixTreeLeafNode::getEndIdx() const {
unsigned SuffixTreeLeafNode::getSuffixIdx() const { return SuffixIdx; }
void SuffixTreeLeafNode::setSuffixIdx(unsigned Idx) { SuffixIdx = Idx; }
+
+unsigned SuffixTreeNode::getLeftLeafIdx() const { return LeftLeafIdx; }
+unsigned SuffixTreeNode::getRightLeafIdx() const { return RightLeafIdx; }
+void SuffixTreeNode::setLeftLeafIdx(unsigned Idx) { LeftLeafIdx = Idx; }
+void SuffixTreeNode::setRightLeafIdx(unsigned Idx) { RightLeafIdx = Idx; }
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-cfi-tail-some.mir b/llvm/test/CodeGen/AArch64/machine-outliner-cfi-tail-some.mir
index 67d411962ce4f7..3afa1d5559a585 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-cfi-tail-some.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-cfi-tail-some.mir
@@ -1,5 +1,5 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=aarch64-apple-unknown -run-pass=machine-outliner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64-apple-unknown -run-pass=machine-outliner -verify-machineinstrs -outliner-leaf-descendants=false %s -o - | FileCheck %s
# Outlining CFI instructions is unsafe if we cannot outline all of the CFI
# instructions from a function. This shows that we choose not to outline the
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-leaf-descendants.ll b/llvm/test/CodeGen/AArch64/machine-outliner-leaf-descendants.ll
new file mode 100644
index 00000000000000..bdaf653a79566d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-leaf-descendants.ll
@@ -0,0 +1,124 @@
+; This test is mainly for the -outliner-leaf-descendants flag for MachineOutliner.
+;
+; ===================== -outliner-leaf-descendants=false =====================
+; MachineOutliner finds THREE key `OutlinedFunction` and outlines them. They are:
+; ```
+; mov w0, #1
+; mov w1, #2
+; mov w2, #3
+; mov w3, #4
+; mov w4, #5
+; mov w5, #6 or #7 or #8
+; b
+; ```
+; Each has:
+; - `SequenceSize=28` and `OccurrenceCount=2`
+; - each Candidate has `CallOverhead=4` and `FrameOverhead=0`
+; - `NotOutlinedCost=28*2=56` and `OutliningCost=4*2+28+0=36`
+; - `Benefit=56-36=20` and `Priority=56/36=1.56`
+;
+; ===================== -outliner-leaf-descendants=false =====================
+; MachineOutliner finds a FOURTH key `OutlinedFunction`, which is:
+; ```
+; mov w0, #1
+; mov w1, #2
+; mov w2, #3
+; mov w3, #4
+; mov w4, #5
+; ```
+; This corresponds to an internal node that has ZERO leaf children, but SIX leaf descendants.
+; It has:
+; - `SequenceSize=20` and `OccurrenceCount=6`
+; - each Candidate has `CallOverhead=12` and `FrameOverhead=4`
+; - `NotOutlinedCost=20*6=120` and `OutliningCost=12*6+20+4=96`
+; - `Benefit=120-96=24` and `Priority=120/96=1.25`
+;
+; The FOURTH `OutlinedFunction` has lower _priority_ compared to the first THREE `OutlinedFunction`
+; Hence, if we additionally include the `-sort-per-priority` flag, the first THREE `OutlinedFunction` are outlined.
+
+; RUN: llc %s -enable-machine-outliner=always -outliner-leaf-descendants=false -filetype=obj -o %t
+; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-BASELINE
+
+; RUN: llc %s -enable-machine-outliner=always -outliner-leaf-descendants=false -outliner-benefit-threshold=22 -filetype=obj -o %t
+; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-NO-CANDIDATE
+
+; RUN: llc %s -enable-machine-outliner=always -outliner-leaf-descendants=true -filetype=obj -o %t
+; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-BASELINE
+
+; RUN: llc %s -enable-machine-outliner=always -outliner-leaf-descendants=true -outliner-benefit-threshold=22 -filetype=obj -o %t
+; RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=CHECK-LEAF-DESCENDANTS
+
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-macosx14.0.0"
+
+declare i32 @_Z3fooiiii(i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef, i32 noundef)
+
+define i32 @_Z2f1v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6)
+ ret i32 %1
+}
+
+define i32 @_Z2f2v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 6)
+ ret i32 %1
+}
+
+define i32 @_Z2f3v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 7)
+ ret i32 %1
+}
+
+define i32 @_Z2f4v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 7)
+ ret i32 %1
+}
+
+define i32 @_Z2f5v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 8)
+ ret i32 %1
+}
+
+define i32 @_Z2f6v() minsize {
+ %1 = tail call i32 @_Z3fooiiii(i32 noundef 1, i32 noundef 2, i32 noundef 3, i32 noundef 4, i32 noundef 5, i32 noundef 8)
+ ret i32 %1
+}
+
+; CHECK-BASELINE: <_OUTLINED_FUNCTION_0>:
+; CHECK-BASELINE-NEXT: mov w0, #0x1
+; CHECK-BASELINE-NEXT: mov w1, #0x2
+; CHECK-BASELINE-NEXT: mov w2, #0x3
+; CHECK-BASELINE-NEXT: mov w3, #0x4
+; CHECK-BASELINE-NEXT: mov w4, #0x5
+; CHECK-BASELINE-NEXT: mov w5, #0x6
+; CHECK-BASELINE-NEXT: b
+
+; CHECK-BASELINE: <_OUTLINED_FUNCTION_1>:
+; CHECK-BASELINE-NEXT: mov w0, #0x1
+; CHECK-BASELINE-NEXT: mov w1, #0x2
+; CHECK-BASELINE-NEXT: mov w2, #0x3
+; CHECK-BASELINE-NEXT: mov w3, #0x4
+; CHECK-BASELINE-NEXT: mov w4, #0x5
+; CHECK-BASELINE-NEXT: mov w5, #0x8
+; CHECK-BASELINE-NEXT: b
+
+; CHECK-BASELINE: <_OUTLINED_FUNCTION_2>:
+; CHECK-BASELINE-NEXT: mov w0, #0x1
+; CHECK-BASELINE-NEXT: mov w1, #0x2
+; CHECK-BASELINE-NEXT: mov w2, #0x3
+; CHECK-BASELINE-NEXT: mov w3, #0x4
+; CHECK-BASELINE-NEXT: mov w4, #0x5
+; CHECK-BASELINE-NEXT: mov w5, #0x7
+; CHECK-BASELINE-NEXT: b
+
+; CHECK-LEAF-DESCENDANTS: <_OUTLINED_FUNCTION_0>:
+; CHECK-LEAF-DESCENDANTS-NEXT: mov w0, #0x1
+; CHECK-LEAF-DESCENDANTS-NEXT: mov w1, #0x2
+; CHECK-LEAF-DESCENDANTS-NEXT: mov w2, #0x3
+; CHECK-LEAF-DESCENDANTS-NEXT: mov w3, #0x4
+; CHECK-LEAF-DESCENDANTS-NEXT: mov w4, #0x5
+; CHECK-LEAF-DESCENDANTS-NEXT: ret
+
+; CHECK-LEAF-DESCENDANTS-NOT: <_OUTLINED_FUNCTION_1>:
+
+; CHECK-NO-CANDIDATE-NOT: <_OUTLINED_FUNCTION_0>:
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir b/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir
index c6bd4c1d04d871..8b0e03dbee8d8d 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-overlap.mir
@@ -1,5 +1,6 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --include-generated-funcs
-# RUN: llc %s -mtriple aarch64 -debug-only=machine-outliner -run-pass=machine-outliner -o - 2>&1 | FileCheck %s
+# RUN: llc %s -mtriple aarch64 -outliner-leaf-descendants=false -debug-only=machine-outliner -run-pass=machine-outliner -o - 2>&1 | FileCheck %s
+# RUN: llc %s -mtriple aarch64 -debug-only=machine-outliner -run-pass=machine-outliner -o - 2>&1 | FileCheck %s --check-prefix=CHECK-LEAF
# REQUIRES: asserts
# CHECK: *** Discarding overlapping candidates ***
@@ -54,6 +55,27 @@ body: |
; CHECK-NEXT: BL @OUTLINED_FUNCTION_0, implicit-def $lr, implicit $sp, implicit-def $lr, implicit-def $x8, implicit-def $x9, implicit $sp, implicit $x0, implicit $x9
; CHECK-NEXT: RET undef $x9
+ ; CHECK-LEAF-LABEL: name: overlap
+ ; CHECK-LEAF: liveins: $x0, $x9
+ ; CHECK-LEAF-NEXT: {{ $}}
+ ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0
+ ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0
+ ; CHECK-LEAF-NEXT: $x8 = ADDXri $x0, 3, 0
+ ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0
+ ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0
+ ; CHECK-LEAF-NEXT: $x8 = ADDXri $x0, 3, 0
+ ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0
+ ; CHECK-LEAF-NEXT: BL @OUTLINED_FUNCTION_0
+ ; CHECK-LEAF-NEXT: RET undef $x9
+
+ ; CHECK-LEAF-LABEL: name: OUTLINED_FUNCTION_0
+ ; CHECK-LEAF: liveins: $x0, $x9, $lr
+ ; CHECK-LEAF-NEXT: {{ $}}
+ ; CHECK-LEAF-NEXT: $x9 = ADDXri $x9, 16, 0
+ ; CHECK-LEAF-NEXT: $x9 = ADDXri $x9, 16, 0
+ ; CHECK-LEAF-NEXT: $x9 = ADDXri $x9, 16, 0
+ ; CHECK-LEAF-NEXT: RET $lr
+
; fixme: outline!
$x9 = ADDXri $x9, 16, 0
$x9 = ADDXri $x9, 16, 0
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.mir b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.mir
index c1c2720dec6ad6..22e5edef2a9395 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-sp-mod.mir
@@ -1,4 +1,4 @@
-# RUN: llc -verify-machineinstrs -run-pass=machine-outliner -run-pass=aarch64-ptrauth %s -o - | FileCheck %s
+# RUN: llc -verify-machineinstrs -run-pass=machine-outliner -run-pass=aarch64-ptrauth -outliner-leaf-descendants=false %s -o - | FileCheck %s
--- |
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
index 9250718fc0d585..618973b9368d1d 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-retaddr-sign-thunk.ll
@@ -1,6 +1,6 @@
-; RUN: llc -mtriple aarch64-arm-linux-gnu --enable-machine-outliner \
+; RUN: llc -mtriple aarch64-arm-linux-gnu --enable-machine-outliner -outliner-leaf-descendants=false \
; RUN: -verify-machineinstrs %s -o - | FileCheck --check-prefixes CHECK,V8A %s
-; RUN-V83A: llc -mtriple aarch64 -enable-machine-outliner \
+; RUN-V83A: llc -mtriple aarch64 -enable-machine-outliner -outliner-leaf-descendants=false \
; RUN-V83A: -verify-machineinstrs -mattr=+v8.3a %s -o - > %t
; RUN-V83A: FileCheck --check-prefixes CHECK,V83A < %t %s
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll b/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll
index aa6e31d6ff21d7..538e1165e39c1d 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-throw2.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64 -frame-pointer=non-leaf < %s | FileCheck %s --check-prefix=NOOMIT
-; RUN: llc -verify-machineinstrs -enable-machine-outliner -mtriple=aarch64 -frame-pointer=none < %s | FileCheck %s --check-prefix=OMITFP
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -outliner-leaf-descendants=false -mtriple=aarch64 -frame-pointer=non-leaf < %s | FileCheck %s --check-prefix=NOOMIT
+; RUN: llc -verify-machineinstrs -enable-machine-outliner -outliner-leaf-descendants=false -mtriple=aarch64 -frame-pointer=none < %s | FileCheck %s --check-prefix=OMITFP
define void @_Z1giii(i32 %x, i32 %y, i32 %z) minsize {
; NOOMIT-LABEL: _Z1giii:
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner-thunk.ll b/llvm/test/CodeGen/AArch64/machine-outliner-thunk.ll
index 8740aac0549eec..7e34adf16d25d3 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner-thunk.ll
+++ b/llvm/test/CodeGen/AArch64/machine-outliner-thunk.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -enable-machine-outliner -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -enable-machine-outliner -outliner-leaf-descendants=false -verify-machineinstrs | FileCheck %s
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-pc-linux-gnu"
diff --git a/llvm/test/CodeGen/AArch64/machine-outliner.mir b/llvm/test/CodeGen/AArch64/machine-outliner.mir
index 83eda744d24a9e..66779addaff0a8 100644
--- a/llvm/test/CodeGen/AArch64/machine-outliner.mir
+++ b/llvm/test/CodeGen/AArch64/machine-outliner.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=aarch64--- -run-pass=prologepilog -run-pass=machine-outliner -verify-machineinstrs -frame-pointer=non-leaf %s -o - | FileCheck %s
+# RUN: llc -mtriple=aarch64--- -run-pass=prologepilog -run-pass=machine-outliner -verify-machineinstrs -frame-pointer=non-leaf -outliner-leaf-descendants=false %s -o - | FileCheck %s
--- |
@x = common global i32 0, align 4
diff --git a/llvm/test/CodeGen/RISCV/machineoutliner-pcrel-lo.mir b/llvm/test/CodeGen/RISCV/machineoutliner-pcrel-lo.mir
index 34f7a93b6a168c..8a83543b0280fd 100644
--- a/llvm/test/CodeGen/RISCV/machineoutliner-pcrel-lo.mir
+++ b/llvm/test/CodeGen/RISCV/machineoutliner-pcrel-lo.mir
@@ -1,11 +1,11 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=riscv32 -x mir -run-pass=machine-outliner -simplify-mir -verify-machineinstrs < %s \
+# RUN: llc -mtriple=riscv32 -x mir -run-pass=machine-outliner -simplify-mir -verify-machineinstrs -outliner-leaf-descendants=false < %s \
# RUN: | FileCheck %s
-# RUN: llc -mtriple=riscv64 -x mir -run-pass=machine-outliner -simplify-mir -verify-machineinstrs < %s \
+# RUN: llc -mtriple=riscv64 -x mir -run-pass=machine-outliner -simplify-mir -verify-machineinstrs -outliner-leaf-descendants=false < %s \
# RUN: | FileCheck %s
-# RUN: llc -mtriple=riscv32 -x mir -run-pass=machine-outliner -simplify-mir --function-sections -verify-machineinstrs < %s \
+# RUN: llc -mtriple=riscv32 -x mir -run-pass=machine-outliner -simplify-mir --function-sections -verify-machineinstrs -outliner-leaf-descendants=false < %s \
# RUN: | FileCheck -check-prefix=CHECK-FS %s
-# RUN: llc -mtriple=riscv64 -x mir -run-pass=machine-outliner -simplify-mir --function-sections -verify-machineinstrs < %s \
+# RUN: llc -mtriple=riscv64 -x mir -run-pass=machine-outliner -simplify-mir --function-sections -verify-machineinstrs -outliner-leaf-descendants=false < %s \
# RUN: | FileCheck -check-prefix=CHECK-FS %s
--- |
diff --git a/llvm/unittests/Support/SuffixTreeTest.cpp b/llvm/unittests/Support/SuffixTreeTest.cpp
index f5d8112ccf5cec..59f6dc9bf963fa 100644
--- a/llvm/unittests/Support/SuffixTreeTest.cpp
+++ b/llvm/unittests/Support/SuffixTreeTest.cpp
@@ -140,4 +140,138 @@ TEST(SuffixTreeTest, TestExclusion) {
}
}
+// Tests that the SuffixTree is able to find the following substrings:
+// {1, 1} at indices 0, 1, 2, 3, and 4;
+// {1, 1, 1} at indices 0, 1, 2, and 3;
+// {1, 1, 1, 1} at indices 0, 1, and 2; and
+// {1, 1, 1, 1, 1} at indices 0 and 1.
+//
+// This is a FIX to the Test TestSingleCharacterRepeat
+TEST(SuffixTreeTest, TestSingleCharacterRepeatWithLeafDescendants) {
+ std::vector<unsigned> RepeatedRepetitionData = {1, 1, 1, 1, 1, 1, 2};
+ std::vector<unsigned>::iterator RRDIt, RRDIt2;
+ SuffixTree ST(RepeatedRepetitionData, true);
+ std::vector<SuffixTree::RepeatedSubstring> SubStrings;
+ for (auto It = ST.begin(); It != ST.end(); It++)
+ SubStrings.push_back(*It);
+ EXPECT_EQ(SubStrings.size(), 4u);
+ for (SuffixTree::RepeatedSubstring &RS : SubStrings) {
+ EXPECT_EQ(RS.StartIndices.size(),
+ RepeatedRepetitionData.size() - RS.Length);
+ for (unsigned StartIdx : SubStrings[0].StartIndices) {
+ RRDIt = RRDIt2 = RepeatedRepetitionData.begin();
+ std::advance(RRDIt, StartIdx);
+ std::advance(RRDIt2, StartIdx + SubStrings[0].Length);
+ ASSERT_TRUE(
+ all_of(make_range<std::vector<unsigned>::iterator>(RRDIt, RRDIt2),
+ [](unsigned Elt) { return Elt == 1; }));
+ }
+ }
+}
+
+// Tests that the SuffixTree is able to find three substrings
+// {1, 2, 3} at indices 6 and 10;
+// {2, 3} at indices 7 and 11; and
+// {1, 2} at indicies 0 and 3.
+//
+// FIXME: {1, 2} has indices 6 and 10 missing as it is a substring of {1, 2, 3}
+// See Test TestSubstringRepeatsWithLeafDescendants for the FIX
+TEST(SuffixTreeTest, TestSubstringRepeats) {
+ std::vector<unsigned> RepeatedRepetitionData = {1, 2, 100, 1, 2, 101, 1,
+ 2, 3, 103, 1, 2, 3, 104};
+ SuffixTree ST(RepeatedRepetitionData);
+ std::vector<SuffixTree::RepeatedSubstring> SubStrings;
+ for (auto It = ST.begin(); It != ST.end(); It++)
+ SubStrings.push_back(*It);
+ EXPECT_EQ(SubStrings.size(), 3u);
+ unsigned Len;
+ for (SuffixTree::RepeatedSubstring &RS : SubStrings) {
+ Len = RS.Length;
+ bool IsExpectedLen = (Len == 3u || Len == 2u);
+ ASSERT_TRUE(IsExpectedLen);
+ bool IsExpectedIndex;
+
+ if (Len == 3u) { // {1, 2, 3}
+ EXPECT_EQ(RS.StartIndices.size(), 2u);
+ for (unsigned StartIdx : RS.StartIndices) {
+ IsExpectedIndex = (StartIdx == 6u || StartIdx == 10u);
+ EXPECT_TRUE(IsExpectedIndex);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx], 1u);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 2u);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx + 2], 3u);
+ }
+ } else {
+ if (RepeatedRepetitionData[RS.StartIndices[0]] == 1u) { // {1, 2}
+ EXPECT_EQ(RS.StartIndices.size(), 2u);
+ for (unsigned StartIdx : RS.StartIndices) {
+ IsExpectedIndex = (StartIdx == 0u || StartIdx == 3u);
+ EXPECT_TRUE(IsExpectedIndex);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 2u);
+ }
+ } else { // {2, 3}
+ EXPECT_EQ(RS.StartIndices.size(), 2u);
+ for (unsigned StartIdx : RS.StartIndices) {
+ IsExpectedIndex = (StartIdx == 7u || StartIdx == 11u);
+ EXPECT_TRUE(IsExpectedIndex);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx], 2u);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 3u);
+ }
+ }
+ }
+ }
+}
+
+// Tests that the SuffixTree is able to find three substrings
+// {1, 2, 3} at indices 6 and 10;
+// {2, 3} at indices 7 and 11; and
+// {1, 2} at indicies 0, 3, 6, and 10.
+//
+// This is a FIX to the Test TestSubstringRepeats
+
+TEST(SuffixTreeTest, TestSubstringRepeatsWithLeafDescendants) {
+ std::vector<unsigned> RepeatedRepetitionData = {1, 2, 100, 1, 2, 101, 1,
+ 2, 3, 103, 1, 2, 3, 104};
+ SuffixTree ST(RepeatedRepetitionData, true);
+ std::vector<SuffixTree::RepeatedSubstring> SubStrings;
+ for (auto It = ST.begin(); It != ST.end(); It++)
+ SubStrings.push_back(*It);
+ EXPECT_EQ(SubStrings.size(), 3u);
+ unsigned Len;
+ for (SuffixTree::RepeatedSubstring &RS : SubStrings) {
+ Len = RS.Length;
+ bool IsExpectedLen = (Len == 3u || Len == 2u);
+ ASSERT_TRUE(IsExpectedLen);
+ bool IsExpectedIndex;
+
+ if (Len == 3u) { // {1, 2, 3}
+ EXPECT_EQ(RS.StartIndices.size(), 2u);
+ for (unsigned StartIdx : RS.StartIndices) {
+ IsExpectedIndex = (StartIdx == 6u || StartIdx == 10u);
+ EXPECT_TRUE(IsExpectedIndex);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx], 1u);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 2u);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx + 2], 3u);
+ }
+ } else {
+ if (RepeatedRepetitionData[RS.StartIndices[0]] == 1u) { // {1, 2}
+ EXPECT_EQ(RS.StartIndices.size(), 4u);
+ for (unsigned StartIdx : RS.StartIndices) {
+ IsExpectedIndex = (StartIdx == 0u || StartIdx == 3u ||
+ StartIdx == 6u || StartIdx == 10u);
+ EXPECT_TRUE(IsExpectedIndex);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 2u);
+ }
+ } else { // {2, 3}
+ EXPECT_EQ(RS.StartIndices.size(), 2u);
+ for (unsigned StartIdx : RS.StartIndices) {
+ IsExpectedIndex = (StartIdx == 7u || StartIdx == 11u);
+ EXPECT_TRUE(IsExpectedIndex);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx], 2u);
+ EXPECT_EQ(RepeatedRepetitionData[StartIdx + 1], 3u);
+ }
+ }
+ }
+ }
+}
+
} // namespace
>From 2c2601941e1b532c121b53d522541f5316a38db9 Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee at meta.com>
Date: Mon, 22 Apr 2024 15:29:25 -0700
Subject: [PATCH 4/9] [CGData] Outlined Hash Tree
This defines the OutlinedHashTree class.
It contains sequences of stable hash values of instructions that have been outlined.
This OutlinedHashTree can be used to track the outlined instruction sequences across modules.
A trie structure is used in its implementation, allowing for a compact sharing of common prefixes.
---
.../llvm/CodeGenData/OutlinedHashTree.h | 107 +++++++++++
.../llvm/CodeGenData/OutlinedHashTreeRecord.h | 67 +++++++
llvm/lib/CMakeLists.txt | 1 +
llvm/lib/CodeGenData/CMakeLists.txt | 14 ++
llvm/lib/CodeGenData/OutlinedHashTree.cpp | 131 ++++++++++++++
.../CodeGenData/OutlinedHashTreeRecord.cpp | 168 ++++++++++++++++++
llvm/unittests/CMakeLists.txt | 1 +
llvm/unittests/CodeGenData/CMakeLists.txt | 14 ++
.../OutlinedHashTreeRecordTest.cpp | 118 ++++++++++++
.../CodeGenData/OutlinedHashTreeTest.cpp | 81 +++++++++
10 files changed, 702 insertions(+)
create mode 100644 llvm/include/llvm/CodeGenData/OutlinedHashTree.h
create mode 100644 llvm/include/llvm/CodeGenData/OutlinedHashTreeRecord.h
create mode 100644 llvm/lib/CodeGenData/CMakeLists.txt
create mode 100644 llvm/lib/CodeGenData/OutlinedHashTree.cpp
create mode 100644 llvm/lib/CodeGenData/OutlinedHashTreeRecord.cpp
create mode 100644 llvm/unittests/CodeGenData/CMakeLists.txt
create mode 100644 llvm/unittests/CodeGenData/OutlinedHashTreeRecordTest.cpp
create mode 100644 llvm/unittests/CodeGenData/OutlinedHashTreeTest.cpp
diff --git a/llvm/include/llvm/CodeGenData/OutlinedHashTree.h b/llvm/include/llvm/CodeGenData/OutlinedHashTree.h
new file mode 100644
index 00000000000000..875e1a78bb4010
--- /dev/null
+++ b/llvm/include/llvm/CodeGenData/OutlinedHashTree.h
@@ -0,0 +1,107 @@
+//===- OutlinedHashTree.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This defines the OutlinedHashTree class. It contains sequences of stable
+// hash values of instructions that have been outlined. This OutlinedHashTree
+// can be used to track the outlined instruction sequences across modules.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGENDATA_OUTLINEDHASHTREE_H
+#define LLVM_CODEGENDATA_OUTLINEDHASHTREE_H
+
+#include "llvm/ADT/StableHashing.h"
+#include "llvm/ObjectYAML/YAML.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <unordered_map>
+#include <vector>
+
+namespace llvm {
+
+/// A HashNode is an entry in an OutlinedHashTree, holding a hash value
+/// and a collection of Successors (other HashNodes). If a HashNode has
+/// a positive terminal value (Terminals > 0), it signifies the end of
+/// a hash sequence with that occurrence count.
+struct HashNode {
+ /// The hash value of the node.
+ stable_hash Hash;
+ /// The number of terminals in the sequence ending at this node.
+ unsigned Terminals;
+ /// The successors of this node.
+ std::unordered_map<stable_hash, std::unique_ptr<HashNode>> Successors;
+};
+
+/// HashNodeStable is the serialized, stable, and compact representation
+/// of a HashNode.
+struct HashNodeStable {
+ llvm::yaml::Hex64 Hash;
+ unsigned Terminals;
+ std::vector<unsigned> SuccessorIds;
+};
+
+class OutlinedHashTree {
+
+ using EdgeCallbackFn =
+ std::function<void(const HashNode *, const HashNode *)>;
+ using NodeCallbackFn = std::function<void(const HashNode *)>;
+
+ using HashSequence = std::vector<stable_hash>;
+ using HashSequencePair = std::pair<std::vector<stable_hash>, unsigned>;
+
+public:
+ /// Walks every edge and node in the OutlinedHashTree and calls CallbackEdge
+ /// for the edges and CallbackNode for the nodes with the stable_hash for
+ /// the source and the stable_hash of the sink for an edge. These generic
+ /// callbacks can be used to traverse a OutlinedHashTree for the purpose of
+ /// print debugging or serializing it.
+ void walkGraph(NodeCallbackFn CallbackNode,
+ EdgeCallbackFn CallbackEdge = nullptr,
+ bool SortedWalk = false) const;
+
+ /// Release all hash nodes except the root hash node.
+ void clear() {
+ assert(getRoot()->Hash == 0 && getRoot()->Terminals == 0);
+ getRoot()->Successors.clear();
+ }
+
+ /// \returns true if the hash tree has only the root node.
+ bool empty() { return size() == 1; }
+
+ /// \returns the size of a OutlinedHashTree by traversing it. If
+ /// \p GetTerminalCountOnly is true, it only counts the terminal nodes
+ /// (meaning it returns the the number of hash sequences in the
+ /// OutlinedHashTree).
+ size_t size(bool GetTerminalCountOnly = false) const;
+
+ /// \returns the depth of a OutlinedHashTree by traversing it.
+ size_t depth() const;
+
+ /// \returns the root hash node of a OutlinedHashTree.
+ const HashNode *getRoot() const { return Root.get(); }
+ HashNode *getRoot() { return Root.get(); }
+
+ /// Inserts a \p Sequence into the this tree. The last node in the sequence
+ /// will increase Terminals.
+ void insert(const HashSequencePair &SequencePair);
+
+ /// Merge a \p OtherTree into this Tree.
+ void merge(const OutlinedHashTree *OtherTree);
+
+ /// \returns the matching count if \p Sequence exists in the OutlinedHashTree.
+ unsigned find(const HashSequence &Sequence) const;
+
+ OutlinedHashTree() { Root = std::make_unique<HashNode>(); }
+
+private:
+ std::unique_ptr<HashNode> Root;
+};
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/include/llvm/CodeGenData/OutlinedHashTreeRecord.h b/llvm/include/llvm/CodeGenData/OutlinedHashTreeRecord.h
new file mode 100644
index 00000000000000..ccd2ad26dd0871
--- /dev/null
+++ b/llvm/include/llvm/CodeGenData/OutlinedHashTreeRecord.h
@@ -0,0 +1,67 @@
+//===- OutlinedHashTreeRecord.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===---------------------------------------------------------------------===//
+//
+// This defines the OutlinedHashTreeRecord class. This class holds the outlined
+// hash tree for both serialization and deserialization processes. It utilizes
+// two data formats for serialization: raw binary data and YAML.
+// These two formats can be used interchangeably.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGENDATA_OUTLINEDHASHTREERECORD_H
+#define LLVM_CODEGENDATA_OUTLINEDHASHTREERECORD_H
+
+#include "llvm/CodeGenData/OutlinedHashTree.h"
+
+namespace llvm {
+
+using IdHashNodeStableMapTy = std::map<unsigned, HashNodeStable>;
+using IdHashNodeMapTy = std::map<unsigned, HashNode *>;
+using HashNodeIdMapTy = std::unordered_map<const HashNode *, unsigned>;
+
+struct OutlinedHashTreeRecord {
+ std::unique_ptr<OutlinedHashTree> HashTree;
+
+ OutlinedHashTreeRecord() { HashTree = std::make_unique<OutlinedHashTree>(); }
+ OutlinedHashTreeRecord(std::unique_ptr<OutlinedHashTree> HashTree)
+ : HashTree(std::move(HashTree)){};
+
+ /// Serialize the outlined hash tree to a raw_ostream.
+ void serialize(raw_ostream &OS) const;
+ /// Deserialize the outlined hash tree from a raw_ostream.
+ void deserialize(const unsigned char *&Ptr);
+ /// Serialize the outlined hash tree to a YAML stream.
+ void serializeYAML(yaml::Output &YOS) const;
+ /// Deserialize the outlined hash tree from a YAML stream.
+ void deserializeYAML(yaml::Input &YIS);
+
+ /// Merge the other outlined hash tree into this one.
+ void merge(const OutlinedHashTreeRecord &Other) {
+ HashTree->merge(Other.HashTree.get());
+ }
+
+ /// \returns true if the outlined hash tree is empty.
+ bool empty() const { return HashTree->empty(); }
+
+ /// Print the outlined hash tree in a YAML format.
+ void print(raw_ostream &OS = llvm::errs()) const {
+ yaml::Output YOS(OS);
+ serializeYAML(YOS);
+ }
+
+private:
+ /// Convert the outlined hash tree to stable data.
+ void convertToStableData(IdHashNodeStableMapTy &IdNodeStableMap) const;
+
+ /// Convert the stable data back to the outlined hash tree.
+ void convertFromStableData(const IdHashNodeStableMapTy &IdNodeStableMap);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGENDATA_OUTLINEDHASHTREERECORD_H
diff --git a/llvm/lib/CMakeLists.txt b/llvm/lib/CMakeLists.txt
index 74e2d03c07953d..2ac0b0dc026e16 100644
--- a/llvm/lib/CMakeLists.txt
+++ b/llvm/lib/CMakeLists.txt
@@ -10,6 +10,7 @@ add_subdirectory(InterfaceStub)
add_subdirectory(IRPrinter)
add_subdirectory(IRReader)
add_subdirectory(CodeGen)
+add_subdirectory(CodeGenData)
add_subdirectory(CodeGenTypes)
add_subdirectory(BinaryFormat)
add_subdirectory(Bitcode)
diff --git a/llvm/lib/CodeGenData/CMakeLists.txt b/llvm/lib/CodeGenData/CMakeLists.txt
new file mode 100644
index 00000000000000..3ba90f96cc86d4
--- /dev/null
+++ b/llvm/lib/CodeGenData/CMakeLists.txt
@@ -0,0 +1,14 @@
+add_llvm_component_library(LLVMCodeGenData
+ OutlinedHashTree.cpp
+ OutlinedHashTreeRecord.cpp
+
+ ADDITIONAL_HEADER_DIRS
+ ${LLVM_MAIN_INCLUDE_DIR}/llvm/CodeGenData
+
+ DEPENDS
+ intrinsics_gen
+
+ LINK_COMPONENTS
+ Core
+ Support
+ )
diff --git a/llvm/lib/CodeGenData/OutlinedHashTree.cpp b/llvm/lib/CodeGenData/OutlinedHashTree.cpp
new file mode 100644
index 00000000000000..032993ded60ead
--- /dev/null
+++ b/llvm/lib/CodeGenData/OutlinedHashTree.cpp
@@ -0,0 +1,131 @@
+//===-- OutlinedHashTree.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// An OutlinedHashTree is a Trie that contains sequences of stable hash values
+// of instructions that have been outlined. This OutlinedHashTree can be used
+// to understand the outlined instruction sequences collected across modules.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGenData/OutlinedHashTree.h"
+
+#include <stack>
+#include <tuple>
+
+#define DEBUG_TYPE "outlined-hash-tree"
+
+using namespace llvm;
+
+void OutlinedHashTree::walkGraph(NodeCallbackFn CallbackNode,
+ EdgeCallbackFn CallbackEdge,
+ bool SortedWalk) const {
+ std::stack<const HashNode *> Stack;
+ Stack.push(getRoot());
+
+ while (!Stack.empty()) {
+ const auto *Current = Stack.top();
+ Stack.pop();
+ if (CallbackNode)
+ CallbackNode(Current);
+
+ auto HandleNext = [&](const HashNode *Next) {
+ if (CallbackEdge)
+ CallbackEdge(Current, Next);
+ Stack.push(Next);
+ };
+ if (SortedWalk) {
+ std::map<stable_hash, const HashNode *> SortedSuccessors;
+ for (const auto &P : Current->Successors)
+ SortedSuccessors[P.first] = P.second.get();
+ for (const auto &P : SortedSuccessors)
+ HandleNext(P.second);
+ } else {
+ for (const auto &P : Current->Successors)
+ HandleNext(P.second.get());
+ }
+ }
+}
+
+size_t OutlinedHashTree::size(bool GetTerminalCountOnly) const {
+ size_t Size = 0;
+ walkGraph([&Size, GetTerminalCountOnly](const HashNode *N) {
+ Size += (N && (!GetTerminalCountOnly || N->Terminals));
+ });
+ return Size;
+}
+
+size_t OutlinedHashTree::depth() const {
+ size_t Size = 0;
+ std::unordered_map<const HashNode *, size_t> DepthMap;
+ walkGraph([&Size, &DepthMap](
+ const HashNode *N) { Size = std::max(Size, DepthMap[N]); },
+ [&DepthMap](const HashNode *Src, const HashNode *Dst) {
+ size_t Depth = DepthMap[Src];
+ DepthMap[Dst] = Depth + 1;
+ });
+ return Size;
+}
+
+void OutlinedHashTree::insert(const HashSequencePair &SequencePair) {
+ const auto &Sequence = SequencePair.first;
+ unsigned Count = SequencePair.second;
+ HashNode *Current = getRoot();
+
+ for (stable_hash StableHash : Sequence) {
+ auto I = Current->Successors.find(StableHash);
+ if (I == Current->Successors.end()) {
+ std::unique_ptr<HashNode> Next = std::make_unique<HashNode>();
+ HashNode *NextPtr = Next.get();
+ NextPtr->Hash = StableHash;
+ Current->Successors.emplace(StableHash, std::move(Next));
+ Current = NextPtr;
+ } else
+ Current = I->second.get();
+ }
+ Current->Terminals += Count;
+}
+
+void OutlinedHashTree::merge(const OutlinedHashTree *Tree) {
+ HashNode *Dst = getRoot();
+ const HashNode *Src = Tree->getRoot();
+ std::stack<std::pair<HashNode *, const HashNode *>> Stack;
+ Stack.push({Dst, Src});
+
+ while (!Stack.empty()) {
+ auto [DstNode, SrcNode] = Stack.top();
+ Stack.pop();
+ if (!SrcNode)
+ continue;
+ DstNode->Terminals += SrcNode->Terminals;
+
+ for (auto &[Hash, NextSrcNode] : SrcNode->Successors) {
+ HashNode *NextDstNode;
+ auto I = DstNode->Successors.find(Hash);
+ if (I == DstNode->Successors.end()) {
+ auto NextDst = std::make_unique<HashNode>();
+ NextDstNode = NextDst.get();
+ NextDstNode->Hash = Hash;
+ DstNode->Successors.emplace(Hash, std::move(NextDst));
+ } else
+ NextDstNode = I->second.get();
+
+ Stack.push({NextDstNode, NextSrcNode.get()});
+ }
+ }
+}
+
+unsigned OutlinedHashTree::find(const HashSequence &Sequence) const {
+ const HashNode *Current = getRoot();
+ for (stable_hash StableHash : Sequence) {
+ const auto I = Current->Successors.find(StableHash);
+ if (I == Current->Successors.end())
+ return 0;
+ Current = I->second.get();
+ }
+ return Current->Terminals;
+}
diff --git a/llvm/lib/CodeGenData/OutlinedHashTreeRecord.cpp b/llvm/lib/CodeGenData/OutlinedHashTreeRecord.cpp
new file mode 100644
index 00000000000000..0d5dd864c89c55
--- /dev/null
+++ b/llvm/lib/CodeGenData/OutlinedHashTreeRecord.cpp
@@ -0,0 +1,168 @@
+//===-- OutlinedHashTreeRecord.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This defines the OutlinedHashTreeRecord class. This class holds the outlined
+// hash tree for both serialization and deserialization processes. It utilizes
+// two data formats for serialization: raw binary data and YAML.
+// These two formats can be used interchangeably.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGenData/OutlinedHashTreeRecord.h"
+#include "llvm/CodeGenData/OutlinedHashTree.h"
+#include "llvm/ObjectYAML/YAML.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/EndianStream.h"
+
+#define DEBUG_TYPE "outlined-hash-tree"
+
+using namespace llvm;
+using namespace llvm::support;
+
+namespace llvm {
+namespace yaml {
+
+template <> struct MappingTraits<HashNodeStable> {
+ static void mapping(IO &io, HashNodeStable &res) {
+ io.mapRequired("Hash", res.Hash);
+ io.mapRequired("Terminals", res.Terminals);
+ io.mapRequired("SuccessorIds", res.SuccessorIds);
+ }
+};
+
+template <> struct CustomMappingTraits<IdHashNodeStableMapTy> {
+ static void inputOne(IO &io, StringRef Key, IdHashNodeStableMapTy &V) {
+ HashNodeStable NodeStable;
+ io.mapRequired(Key.str().c_str(), NodeStable);
+ unsigned Id;
+ if (Key.getAsInteger(0, Id)) {
+ io.setError("Id not an integer");
+ return;
+ }
+ V.insert({Id, NodeStable});
+ }
+
+ static void output(IO &io, IdHashNodeStableMapTy &V) {
+ for (auto Iter = V.begin(); Iter != V.end(); ++Iter)
+ io.mapRequired(utostr(Iter->first).c_str(), Iter->second);
+ }
+};
+
+} // namespace yaml
+} // namespace llvm
+
+void OutlinedHashTreeRecord::serialize(raw_ostream &OS) const {
+ IdHashNodeStableMapTy IdNodeStableMap;
+ convertToStableData(IdNodeStableMap);
+ support::endian::Writer Writer(OS, endianness::little);
+ Writer.write<uint32_t>(IdNodeStableMap.size());
+
+ for (const auto &[Id, NodeStable] : IdNodeStableMap) {
+ Writer.write<uint32_t>(Id);
+ Writer.write<uint64_t>(NodeStable.Hash);
+ Writer.write<uint32_t>(NodeStable.Terminals);
+ Writer.write<uint32_t>(NodeStable.SuccessorIds.size());
+ for (auto SuccessorId : NodeStable.SuccessorIds)
+ Writer.write<uint32_t>(SuccessorId);
+ }
+}
+
+void OutlinedHashTreeRecord::deserialize(const unsigned char *&Ptr) {
+ IdHashNodeStableMapTy IdNodeStableMap;
+ auto NumIdNodeStableMap =
+ endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
+
+ for (unsigned I = 0; I < NumIdNodeStableMap; ++I) {
+ auto Id = endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
+ HashNodeStable NodeStable;
+ NodeStable.Hash =
+ endian::readNext<uint64_t, endianness::little, unaligned>(Ptr);
+ NodeStable.Terminals =
+ endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
+ auto NumSuccessorIds =
+ endian::readNext<uint32_t, endianness::little, unaligned>(Ptr);
+ for (unsigned J = 0; J < NumSuccessorIds; ++J)
+ NodeStable.SuccessorIds.push_back(
+ endian::readNext<uint32_t, endianness::little, unaligned>(Ptr));
+
+ IdNodeStableMap[Id] = std::move(NodeStable);
+ }
+
+ convertFromStableData(IdNodeStableMap);
+}
+
+void OutlinedHashTreeRecord::serializeYAML(yaml::Output &YOS) const {
+ IdHashNodeStableMapTy IdNodeStableMap;
+ convertToStableData(IdNodeStableMap);
+
+ YOS << IdNodeStableMap;
+}
+
+void OutlinedHashTreeRecord::deserializeYAML(yaml::Input &YIS) {
+ IdHashNodeStableMapTy IdNodeStableMap;
+
+ YIS >> IdNodeStableMap;
+ YIS.nextDocument();
+
+ convertFromStableData(IdNodeStableMap);
+}
+
+void OutlinedHashTreeRecord::convertToStableData(
+ IdHashNodeStableMapTy &IdNodeStableMap) const {
+ // Build NodeIdMap
+ HashNodeIdMapTy NodeIdMap;
+ HashTree->walkGraph(
+ [&NodeIdMap](const HashNode *Current) {
+ size_t Index = NodeIdMap.size();
+ NodeIdMap[Current] = Index;
+ assert(Index = NodeIdMap.size() + 1 &&
+ "Expected size of NodeMap to increment by 1");
+ },
+ /*EdgeCallbackFn=*/nullptr, /*SortedWork=*/true);
+
+ // Convert NodeIdMap to NodeStableMap
+ for (auto &P : NodeIdMap) {
+ auto *Node = P.first;
+ auto Id = P.second;
+ HashNodeStable NodeStable;
+ NodeStable.Hash = Node->Hash;
+ NodeStable.Terminals = Node->Terminals;
+ for (auto &P : Node->Successors)
+ NodeStable.SuccessorIds.push_back(NodeIdMap[P.second.get()]);
+ IdNodeStableMap[Id] = NodeStable;
+ }
+
+ // Sort the Successors so that they come out in the same order as in the map.
+ for (auto &P : IdNodeStableMap)
+ std::sort(P.second.SuccessorIds.begin(), P.second.SuccessorIds.end());
+}
+
+void OutlinedHashTreeRecord::convertFromStableData(
+ const IdHashNodeStableMapTy &IdNodeStableMap) {
+ IdHashNodeMapTy IdNodeMap;
+ // Initialize the root node at 0.
+ IdNodeMap[0] = HashTree->getRoot();
+ assert(IdNodeMap[0]->Successors.empty());
+
+ for (auto &P : IdNodeStableMap) {
+ auto Id = P.first;
+ const HashNodeStable &NodeStable = P.second;
+ assert(IdNodeMap.count(Id));
+ HashNode *Curr = IdNodeMap[Id];
+ Curr->Hash = NodeStable.Hash;
+ Curr->Terminals = NodeStable.Terminals;
+ auto &Successors = Curr->Successors;
+ assert(Successors.empty());
+ for (auto SuccessorId : NodeStable.SuccessorIds) {
+ auto Sucessor = std::make_unique<HashNode>();
+ IdNodeMap[SuccessorId] = Sucessor.get();
+ auto Hash = IdNodeStableMap.at(SuccessorId).Hash;
+ Successors[Hash] = std::move(Sucessor);
+ }
+ }
+}
diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt
index 46f30ff398e10d..cb4b8513e6d02e 100644
--- a/llvm/unittests/CMakeLists.txt
+++ b/llvm/unittests/CMakeLists.txt
@@ -21,6 +21,7 @@ add_subdirectory(BinaryFormat)
add_subdirectory(Bitcode)
add_subdirectory(Bitstream)
add_subdirectory(CodeGen)
+add_subdirectory(CodeGenData)
add_subdirectory(DebugInfo)
add_subdirectory(Debuginfod)
add_subdirectory(Demangle)
diff --git a/llvm/unittests/CodeGenData/CMakeLists.txt b/llvm/unittests/CodeGenData/CMakeLists.txt
new file mode 100644
index 00000000000000..3d821b87e29d8c
--- /dev/null
+++ b/llvm/unittests/CodeGenData/CMakeLists.txt
@@ -0,0 +1,14 @@
+set(LLVM_LINK_COMPONENTS
+ ${LLVM_TARGETS_TO_BUILD}
+ CodeGen
+ CodeGenData
+ Core
+ Support
+ )
+
+add_llvm_unittest(CodeGenDataTests
+ OutlinedHashTreeRecordTest.cpp
+ OutlinedHashTreeTest.cpp
+ )
+
+target_link_libraries(CodeGenDataTests PRIVATE LLVMTestingSupport)
diff --git a/llvm/unittests/CodeGenData/OutlinedHashTreeRecordTest.cpp b/llvm/unittests/CodeGenData/OutlinedHashTreeRecordTest.cpp
new file mode 100644
index 00000000000000..aa7ad4a33754ff
--- /dev/null
+++ b/llvm/unittests/CodeGenData/OutlinedHashTreeRecordTest.cpp
@@ -0,0 +1,118 @@
+//===- OutlinedHashTreeRecordTest.cpp -------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGenData/OutlinedHashTreeRecord.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+TEST(OutlinedHashTreeRecordTest, Empty) {
+ OutlinedHashTreeRecord HashTreeRecord;
+ ASSERT_TRUE(HashTreeRecord.empty());
+}
+
+TEST(OutlinedHashTreeRecordTest, Print) {
+ OutlinedHashTreeRecord HashTreeRecord;
+ HashTreeRecord.HashTree->insert({{1, 2}, 3});
+
+ const char *ExpectedTreeStr = R"(---
+0:
+ Hash: 0x0
+ Terminals: 0
+ SuccessorIds: [ 1 ]
+1:
+ Hash: 0x1
+ Terminals: 0
+ SuccessorIds: [ 2 ]
+2:
+ Hash: 0x2
+ Terminals: 3
+ SuccessorIds: [ ]
+...
+)";
+ std::string TreeDump;
+ raw_string_ostream OS(TreeDump);
+ HashTreeRecord.print(OS);
+ EXPECT_EQ(ExpectedTreeStr, TreeDump);
+}
+
+TEST(OutlinedHashTreeRecordTest, Stable) {
+ OutlinedHashTreeRecord HashTreeRecord1;
+ HashTreeRecord1.HashTree->insert({{1, 2}, 4});
+ HashTreeRecord1.HashTree->insert({{1, 3}, 5});
+
+ OutlinedHashTreeRecord HashTreeRecord2;
+ HashTreeRecord2.HashTree->insert({{1, 3}, 5});
+ HashTreeRecord2.HashTree->insert({{1, 2}, 4});
+
+ // Output is stable regardless of insertion order.
+ std::string TreeDump1;
+ raw_string_ostream OS1(TreeDump1);
+ HashTreeRecord1.print(OS1);
+ std::string TreeDump2;
+ raw_string_ostream OS2(TreeDump2);
+ HashTreeRecord2.print(OS2);
+
+ EXPECT_EQ(TreeDump1, TreeDump2);
+}
+
+TEST(OutlinedHashTreeRecordTest, Serialize) {
+ OutlinedHashTreeRecord HashTreeRecord1;
+ HashTreeRecord1.HashTree->insert({{1, 2}, 4});
+ HashTreeRecord1.HashTree->insert({{1, 3}, 5});
+
+ // Serialize and deserialize the tree.
+ SmallVector<char> Out;
+ raw_svector_ostream OS(Out);
+ HashTreeRecord1.serialize(OS);
+
+ OutlinedHashTreeRecord HashTreeRecord2;
+ const uint8_t *Data = reinterpret_cast<const uint8_t *>(Out.data());
+ HashTreeRecord2.deserialize(Data);
+
+ // Two trees should be identical.
+ std::string TreeDump1;
+ raw_string_ostream OS1(TreeDump1);
+ HashTreeRecord1.print(OS1);
+ std::string TreeDump2;
+ raw_string_ostream OS2(TreeDump2);
+ HashTreeRecord2.print(OS2);
+
+ EXPECT_EQ(TreeDump1, TreeDump2);
+}
+
+TEST(OutlinedHashTreeRecordTest, SerializeYAML) {
+ OutlinedHashTreeRecord HashTreeRecord1;
+ HashTreeRecord1.HashTree->insert({{1, 2}, 4});
+ HashTreeRecord1.HashTree->insert({{1, 3}, 5});
+
+ // Serialize and deserialize the tree in a YAML format.
+ std::string Out;
+ raw_string_ostream OS(Out);
+ yaml::Output YOS(OS);
+ HashTreeRecord1.serializeYAML(YOS);
+
+ OutlinedHashTreeRecord HashTreeRecord2;
+ yaml::Input YIS(StringRef(Out.data(), Out.size()));
+ HashTreeRecord2.deserializeYAML(YIS);
+
+ // Two trees should be identical.
+ std::string TreeDump1;
+ raw_string_ostream OS1(TreeDump1);
+ HashTreeRecord1.print(OS1);
+ std::string TreeDump2;
+ raw_string_ostream OS2(TreeDump2);
+ HashTreeRecord2.print(OS2);
+
+ EXPECT_EQ(TreeDump1, TreeDump2);
+}
+
+} // end namespace
diff --git a/llvm/unittests/CodeGenData/OutlinedHashTreeTest.cpp b/llvm/unittests/CodeGenData/OutlinedHashTreeTest.cpp
new file mode 100644
index 00000000000000..d11618cf8e4fae
--- /dev/null
+++ b/llvm/unittests/CodeGenData/OutlinedHashTreeTest.cpp
@@ -0,0 +1,81 @@
+//===- OutlinedHashTreeTest.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGenData/OutlinedHashTree.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+TEST(OutlinedHashTreeTest, Empty) {
+ OutlinedHashTree HashTree;
+ ASSERT_TRUE(HashTree.empty());
+ // The header node is always present.
+ ASSERT_TRUE(HashTree.size() == 1);
+ ASSERT_TRUE(HashTree.depth() == 0);
+}
+
+TEST(OutlinedHashTreeTest, Insert) {
+ OutlinedHashTree HashTree;
+ HashTree.insert({{1, 2, 3}, 1});
+ // The node count is 4 (including the root node).
+ ASSERT_TRUE(HashTree.size() == 4);
+ // The terminal count is 1.
+ ASSERT_TRUE(HashTree.size(/*GetTerminalCountOnly=*/true) == 1);
+ // The depth is 3.
+ ASSERT_TRUE(HashTree.depth() == 3);
+
+ HashTree.clear();
+ ASSERT_TRUE(HashTree.empty());
+
+ HashTree.insert({{1, 2, 3}, 1});
+ HashTree.insert({{1, 2, 4}, 2});
+ // The nodes of 1 and 2 are shared with the same prefix.
+ // The nodes are root, 1, 2, 3 and 4, whose counts are 5.
+ ASSERT_TRUE(HashTree.size() == 5);
+}
+
+TEST(OutlinedHashTreeTest, Find) {
+ OutlinedHashTree HashTree;
+ HashTree.insert({{1, 2, 3}, 1});
+ HashTree.insert({{1, 2, 3}, 2});
+
+ // The node count does not change as the same sequences are added.
+ ASSERT_TRUE(HashTree.size() == 4);
+ // The terminal counts are accumulated from two same sequences.
+ ASSERT_TRUE(HashTree.find({1, 2, 3}) == 3);
+ ASSERT_TRUE(HashTree.find({1, 2}) == 0);
+}
+
+TEST(OutlinedHashTreeTest, Merge) {
+ // Build HashTree1 inserting 2 sequences.
+ OutlinedHashTree HashTree1;
+
+ HashTree1.insert({{1, 2}, 20});
+ HashTree1.insert({{1, 4}, 30});
+
+ // Build HashTree2 and HashTree3 for each
+ OutlinedHashTree HashTree2;
+ HashTree2.insert({{1, 2}, 20});
+ OutlinedHashTree HashTree3;
+ HashTree3.insert({{1, 4}, 30});
+
+ // Merge HashTree3 into HashTree2.
+ HashTree2.merge(&HashTree3);
+
+ // Compare HashTree1 and HashTree2.
+ EXPECT_EQ(HashTree1.size(), HashTree2.size());
+ EXPECT_EQ(HashTree1.depth(), HashTree2.depth());
+ EXPECT_EQ(HashTree1.find({1, 2}), HashTree2.find({1, 2}));
+ EXPECT_EQ(HashTree1.find({1, 4}), HashTree2.find({1, 4}));
+ EXPECT_EQ(HashTree1.find({1, 3}), HashTree2.find({1, 3}));
+}
+
+} // end namespace
>From ef71699a35d50c4b0ef4aa4f17648536c8236a63 Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee at meta.com>
Date: Tue, 23 Apr 2024 14:22:14 -0700
Subject: [PATCH 5/9] [CGData] llvm-cgdata
The llvm-cgdata tool has been introduced to handle reading and writing of codegen data. This data includes an optimistic codegen summary that can be utilized to enhance subsequent codegen. Currently, the tool supports saving and restoring the outlined hash tree, facilitating machine function outlining across modules. Additional codegen summaries can be incorporated into separate sections as required. This patch primarily establishes basic support for the reader and writer, similar to llvm-profdata.
The high-level operations of llvm-cgdata are as follows:
1. It reads local raw codegen data from a custom section (for example, __llvm_outline) embedded in native binary files
2. It merges local raw codegen data into an indexed codegen data, complete with a suitable header.
3. It handles reading and writing of the indexed codegen data into a standalone file.
---
llvm/include/llvm/CodeGenData/CodeGenData.h | 202 +++++++++++++
llvm/include/llvm/CodeGenData/CodeGenData.inc | 46 +++
.../llvm/CodeGenData/CodeGenDataReader.h | 154 ++++++++++
.../llvm/CodeGenData/CodeGenDataWriter.h | 68 +++++
llvm/lib/CodeGenData/CMakeLists.txt | 3 +
llvm/lib/CodeGenData/CodeGenData.cpp | 197 +++++++++++++
llvm/lib/CodeGenData/CodeGenDataReader.cpp | 174 ++++++++++++
llvm/lib/CodeGenData/CodeGenDataWriter.cpp | 162 +++++++++++
llvm/test/CMakeLists.txt | 1 +
llvm/test/lit.cfg.py | 1 +
llvm/test/tools/llvm-cgdata/dump.test | 30 ++
llvm/test/tools/llvm-cgdata/empty.test | 32 +++
llvm/test/tools/llvm-cgdata/error.test | 38 +++
.../test/tools/llvm-cgdata/merge-archive.test | 75 +++++
llvm/test/tools/llvm-cgdata/merge-concat.test | 68 +++++
llvm/test/tools/llvm-cgdata/merge-double.test | 74 +++++
llvm/test/tools/llvm-cgdata/merge-single.test | 43 +++
llvm/test/tools/llvm-cgdata/show.test | 30 ++
llvm/tools/llvm-cgdata/CMakeLists.txt | 15 +
llvm/tools/llvm-cgdata/llvm-cgdata.cpp | 268 ++++++++++++++++++
20 files changed, 1681 insertions(+)
create mode 100644 llvm/include/llvm/CodeGenData/CodeGenData.h
create mode 100644 llvm/include/llvm/CodeGenData/CodeGenData.inc
create mode 100644 llvm/include/llvm/CodeGenData/CodeGenDataReader.h
create mode 100644 llvm/include/llvm/CodeGenData/CodeGenDataWriter.h
create mode 100644 llvm/lib/CodeGenData/CodeGenData.cpp
create mode 100644 llvm/lib/CodeGenData/CodeGenDataReader.cpp
create mode 100644 llvm/lib/CodeGenData/CodeGenDataWriter.cpp
create mode 100644 llvm/test/tools/llvm-cgdata/dump.test
create mode 100644 llvm/test/tools/llvm-cgdata/empty.test
create mode 100644 llvm/test/tools/llvm-cgdata/error.test
create mode 100644 llvm/test/tools/llvm-cgdata/merge-archive.test
create mode 100644 llvm/test/tools/llvm-cgdata/merge-concat.test
create mode 100644 llvm/test/tools/llvm-cgdata/merge-double.test
create mode 100644 llvm/test/tools/llvm-cgdata/merge-single.test
create mode 100644 llvm/test/tools/llvm-cgdata/show.test
create mode 100644 llvm/tools/llvm-cgdata/CMakeLists.txt
create mode 100644 llvm/tools/llvm-cgdata/llvm-cgdata.cpp
diff --git a/llvm/include/llvm/CodeGenData/CodeGenData.h b/llvm/include/llvm/CodeGenData/CodeGenData.h
new file mode 100644
index 00000000000000..118fb9841d27e8
--- /dev/null
+++ b/llvm/include/llvm/CodeGenData/CodeGenData.h
@@ -0,0 +1,202 @@
+//===- CodeGenData.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for codegen data that has stable summary which
+// can be used to optimize the code in the subsequent codegen.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGENDATA_CODEGENDATA_H
+#define LLVM_CODEGENDATA_CODEGENDATA_H
+
+#include "llvm/ADT/BitmaskEnum.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/CodeGenData/OutlinedHashTree.h"
+#include "llvm/CodeGenData/OutlinedHashTreeRecord.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/TargetParser/Triple.h"
+#include <mutex>
+
+namespace llvm {
+
+enum CGDataSectKind {
+#define CG_DATA_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind,
+#include "llvm/CodeGenData/CodeGenData.inc"
+};
+
+std::string getCodeGenDataSectionName(CGDataSectKind CGSK,
+ Triple::ObjectFormatType OF,
+ bool AddSegmentInfo = true);
+
+enum class CGDataKind {
+ Unknown = 0x0,
+ // A function outlining info.
+ FunctionOutlinedHashTree = 0x1,
+ LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/FunctionOutlinedHashTree)
+};
+
+const std::error_category &cgdata_category();
+
+enum class cgdata_error {
+ success = 0,
+ eof,
+ bad_magic,
+ bad_header,
+ empty_cgdata,
+ malformed,
+ unsupported_version,
+};
+
+inline std::error_code make_error_code(cgdata_error E) {
+ return std::error_code(static_cast<int>(E), cgdata_category());
+}
+
+class CGDataError : public ErrorInfo<CGDataError> {
+public:
+ CGDataError(cgdata_error Err, const Twine &ErrStr = Twine())
+ : Err(Err), Msg(ErrStr.str()) {
+ assert(Err != cgdata_error::success && "Not an error");
+ }
+
+ std::string message() const override;
+
+ void log(raw_ostream &OS) const override { OS << message(); }
+
+ std::error_code convertToErrorCode() const override {
+ return make_error_code(Err);
+ }
+
+ cgdata_error get() const { return Err; }
+ const std::string &getMessage() const { return Msg; }
+
+ /// Consume an Error and return the raw enum value contained within it, and
+ /// the optional error message. The Error must either be a success value, or
+ /// contain a single CGDataError.
+ static std::pair<cgdata_error, std::string> take(Error E) {
+ auto Err = cgdata_error::success;
+ std::string Msg = "";
+ handleAllErrors(std::move(E), [&Err, &Msg](const CGDataError &IPE) {
+ assert(Err == cgdata_error::success && "Multiple errors encountered");
+ Err = IPE.get();
+ Msg = IPE.getMessage();
+ });
+ return {Err, Msg};
+ }
+
+ static char ID;
+
+private:
+ cgdata_error Err;
+ std::string Msg;
+};
+
+enum CGDataMode {
+ None,
+ Read,
+ Write,
+};
+
+class CodeGenData {
+ /// Global outlined hash tree that has oulined hash sequences across modules.
+ std::unique_ptr<OutlinedHashTree> PublishedHashTree;
+
+ /// This flag is set when -fcgdata-generate is passed.
+ /// Or, it can be mutated with -ftwo-codegen-rounds during two codegen runs.
+ bool EmitCGData;
+
+ /// This is a singleton instance which is thread-safe. Unlike profile data
+ /// which is largely function-based, codegen data describes the whole module.
+ /// Therefore, this can be initialized once, and can be used across modules
+ /// instead of constructing the same one for each codegen backend.
+ static std::unique_ptr<CodeGenData> Instance;
+ static std::once_flag OnceFlag;
+
+ CodeGenData() = default;
+
+public:
+ ~CodeGenData() = default;
+
+ static CodeGenData &getInstance();
+
+ /// Returns true if we have a valid outlined hash tree.
+ bool hasOutlinedHashTree() {
+ return PublishedHashTree && !PublishedHashTree->empty();
+ }
+
+ /// Returns the outlined hash tree. This can be globally used in a read-only
+ /// manner.
+ const OutlinedHashTree *getOutlinedHashTree() {
+ return PublishedHashTree.get();
+ }
+
+ /// Returns true if we should write codegen data.
+ bool emitCGData() { return EmitCGData; }
+
+ /// Publish the (globally) merged or read outlined hash tree.
+ void publishOutlinedHashTree(std::unique_ptr<OutlinedHashTree> HashTree) {
+ PublishedHashTree = std::move(HashTree);
+ // Ensure we disable emitCGData as we do not want to read and write both.
+ EmitCGData = false;
+ }
+};
+
+namespace cgdata {
+
+inline bool hasOutlinedHashTree() {
+ return CodeGenData::getInstance().hasOutlinedHashTree();
+}
+
+inline const OutlinedHashTree *getOutlinedHashTree() {
+ return CodeGenData::getInstance().getOutlinedHashTree();
+}
+
+inline bool emitCGData() { return CodeGenData::getInstance().emitCGData(); }
+
+inline void
+publishOutlinedHashTree(std::unique_ptr<OutlinedHashTree> HashTree) {
+ CodeGenData::getInstance().publishOutlinedHashTree(std::move(HashTree));
+}
+
+void warn(Error E, StringRef Whence = "");
+void warn(Twine Message, std::string Whence = "", std::string Hint = "");
+
+} // end namespace cgdata
+
+namespace IndexedCGData {
+
+const uint64_t Magic = 0x81617461646763ff; // "\xffcgdata\x81"
+
+enum CGDataVersion {
+ // Version 1 is the first version. This version support the outlined
+ // hash tree.
+ Version1 = 1,
+ CurrentVersion = CG_DATA_INDEX_VERSION
+};
+const uint64_t Version = CGDataVersion::CurrentVersion;
+
+struct Header {
+ uint64_t Magic;
+ uint32_t Version;
+ uint32_t DataKind;
+ uint64_t OutlinedHashTreeOffset;
+
+ // New fields should only be added at the end to ensure that the size
+ // computation is correct. The methods below need to be updated to ensure that
+ // the new field is read correctly.
+
+ // Reads a header struct from the buffer.
+ static Expected<Header> readFromBuffer(const unsigned char *Curr);
+};
+
+} // end namespace IndexedCGData
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_PREPARE_H
diff --git a/llvm/include/llvm/CodeGenData/CodeGenData.inc b/llvm/include/llvm/CodeGenData/CodeGenData.inc
new file mode 100644
index 00000000000000..5f6df5c0bf1065
--- /dev/null
+++ b/llvm/include/llvm/CodeGenData/CodeGenData.inc
@@ -0,0 +1,46 @@
+/*===-- CodeGenData.inc ----------------------------------------*- C++ -*-=== *\
+|*
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+|*
+\*===----------------------------------------------------------------------===*/
+/*
+ * This is the main file that defines all the data structure, signature,
+ * constant literals that are shared across compiler, host tools (reader/writer)
+ * to support codegen data.
+ *
+\*===----------------------------------------------------------------------===*/
+
+#ifdef CG_DATA_SECT_ENTRY
+#define CG_DATA_DEFINED
+CG_DATA_SECT_ENTRY(CG_outline, CG_DATA_QUOTE(CG_DATA_OUTLINE_COMMON),
+ CG_DATA_OUTLINE_COFF, "__DATA,")
+
+#undef CG_DATA_SECT_ENTRY
+#endif
+
+/* section name strings common to all targets other
+ than WIN32 */
+#define CG_DATA_OUTLINE_COMMON __llvm_outline
+/* Since cg data sections are not allocated, we don't need to
+ * access them at runtime.
+ */
+#define CG_DATA_OUTLINE_COFF ".loutline"
+
+#ifdef _WIN32
+/* Runtime section names and name strings. */
+#define CG_DATA_SECT_NAME CG_DATA_OUTLINE_COFF
+
+#else
+/* Runtime section names and name strings. */
+#define CG_DATA_SECT_NAME INSTR_PROF_QUOTE(CG_DATA_OUTLINE_COMMON)
+
+#endif
+
+/* Indexed codegen data format version (start from 1). */
+#define CG_DATA_INDEX_VERSION 1
+
+/* Helper macros. */
+#define CG_DATA_SIMPLE_QUOTE(x) #x
+#define CG_DATA_QUOTE(x) CG_DATA_SIMPLE_QUOTE(x)
diff --git a/llvm/include/llvm/CodeGenData/CodeGenDataReader.h b/llvm/include/llvm/CodeGenData/CodeGenDataReader.h
new file mode 100644
index 00000000000000..df4ae3ed24e79a
--- /dev/null
+++ b/llvm/include/llvm/CodeGenData/CodeGenDataReader.h
@@ -0,0 +1,154 @@
+//===- CodeGenDataReader.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for reading codegen data.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGENDATA_CODEGENDATAREADER_H
+#define LLVM_CODEGENDATA_CODEGENDATAREADER_H
+
+#include "llvm/CodeGenData/CodeGenData.h"
+#include "llvm/CodeGenData/OutlinedHashTreeRecord.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/VirtualFileSystem.h"
+
+namespace llvm {
+
+class CodeGenDataReader {
+ cgdata_error LastError = cgdata_error::success;
+ std::string LastErrorMsg;
+
+public:
+ CodeGenDataReader() = default;
+ virtual ~CodeGenDataReader() = default;
+
+ /// Read the header. Required before reading first record.
+ virtual Error read() = 0;
+ /// Return the codegen data version.
+ virtual uint32_t getVersion() const = 0;
+ /// Return the codegen data kind.
+ virtual CGDataKind getDataKind() const = 0;
+ /// Return true if the data has an outlined hash tree.
+ virtual bool hasOutlinedHashTree() const = 0;
+ /// Return the outlined hash tree that is released from the reader.
+ std::unique_ptr<OutlinedHashTree> releaseOutlinedHashTree() {
+ return std::move(HashTreeRecord.HashTree);
+ }
+
+ /// Factory method to create an appropriately typed reader for the given
+ /// codegen data file path and file system.
+ static Expected<std::unique_ptr<CodeGenDataReader>>
+ create(const Twine &Path, vfs::FileSystem &FS);
+
+ /// Factory method to create an appropriately typed reader for the given
+ /// memory buffer.
+ static Expected<std::unique_ptr<CodeGenDataReader>>
+ create(std::unique_ptr<MemoryBuffer> Buffer);
+
+ /// Extract the cgdata embedded in sections from the given object file and
+ /// merge them into the GlobalOutlineRecord. This is a static helper that
+ /// is used by `llvm-cgdata merge` or ThinLTO's two-codegen rounds.
+ static Error mergeFromObjectFile(const object::ObjectFile *Obj,
+ OutlinedHashTreeRecord &GlobalOutlineRecord);
+
+protected:
+ /// The outlined hash tree that has been read. When it's released by
+ /// releaseOutlinedHashTree(), it's no longer valid.
+ OutlinedHashTreeRecord HashTreeRecord;
+
+ /// Set the current error and return same.
+ Error error(cgdata_error Err, const std::string &ErrMsg = "") {
+ LastError = Err;
+ LastErrorMsg = ErrMsg;
+ if (Err == cgdata_error::success)
+ return Error::success();
+ return make_error<CGDataError>(Err, ErrMsg);
+ }
+
+ Error error(Error &&E) {
+ handleAllErrors(std::move(E), [&](const CGDataError &IPE) {
+ LastError = IPE.get();
+ LastErrorMsg = IPE.getMessage();
+ });
+ return make_error<CGDataError>(LastError, LastErrorMsg);
+ }
+
+ /// Clear the current error and return a successful one.
+ Error success() { return error(cgdata_error::success); }
+};
+
+class IndexedCodeGenDataReader : public CodeGenDataReader {
+ /// The codegen data file contents.
+ std::unique_ptr<MemoryBuffer> DataBuffer;
+ /// The header
+ IndexedCGData::Header Header;
+
+public:
+ IndexedCodeGenDataReader(std::unique_ptr<MemoryBuffer> DataBuffer)
+ : DataBuffer(std::move(DataBuffer)) {}
+ IndexedCodeGenDataReader(const IndexedCodeGenDataReader &) = delete;
+ IndexedCodeGenDataReader &
+ operator=(const IndexedCodeGenDataReader &) = delete;
+
+ /// Return true if the given buffer is in binary codegen data format.
+ static bool hasFormat(const MemoryBuffer &Buffer);
+ /// Read the contents including the header.
+ Error read() override;
+ /// Return the codegen data version.
+ uint32_t getVersion() const override { return Header.Version; }
+ /// Return the codegen data kind.
+ CGDataKind getDataKind() const override {
+ return static_cast<CGDataKind>(Header.DataKind);
+ }
+ /// Return true if the header indicates the data has an outlined hash tree.
+ /// This does not mean that the data is still available.
+ bool hasOutlinedHashTree() const override {
+ return Header.DataKind &
+ static_cast<uint32_t>(CGDataKind::FunctionOutlinedHashTree);
+ }
+};
+
+/// This format is a simple text format that's suitable for test data.
+/// The header is a custom format starting with `:` per line to indicate which
+/// codegen data is recorded. `#` is used to indicate a comment.
+/// The subsequent data is a YAML format per each codegen data in order.
+/// Currently, it only has a function outlined hash tree.
+class TextCodeGenDataReader : public CodeGenDataReader {
+ /// The codegen data file contents.
+ std::unique_ptr<MemoryBuffer> DataBuffer;
+ /// Iterator over the profile data.
+ line_iterator Line;
+ /// Describe the kind of the codegen data.
+ CGDataKind DataKind = CGDataKind::Unknown;
+
+public:
+ TextCodeGenDataReader(std::unique_ptr<MemoryBuffer> DataBuffer_)
+ : DataBuffer(std::move(DataBuffer_)), Line(*DataBuffer, true, '#') {}
+ TextCodeGenDataReader(const TextCodeGenDataReader &) = delete;
+ TextCodeGenDataReader &operator=(const TextCodeGenDataReader &) = delete;
+
+ /// Return true if the given buffer is in text codegen data format.
+ static bool hasFormat(const MemoryBuffer &Buffer);
+ /// Read the contents including the header.
+ Error read() override;
+ /// Text format does not have version, so return 0.
+ uint32_t getVersion() const override { return 0; }
+ /// Return the codegen data kind.
+ CGDataKind getDataKind() const override { return DataKind; }
+ /// Return true if the header indicates the data has an outlined hash tree.
+ /// This does not mean that the data is still available.
+ bool hasOutlinedHashTree() const override {
+ return static_cast<uint32_t>(DataKind) &
+ static_cast<uint32_t>(CGDataKind::FunctionOutlinedHashTree);
+ }
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGENDATA_CODEGENDATAREADER_H
diff --git a/llvm/include/llvm/CodeGenData/CodeGenDataWriter.h b/llvm/include/llvm/CodeGenData/CodeGenDataWriter.h
new file mode 100644
index 00000000000000..e17ffc3482ec91
--- /dev/null
+++ b/llvm/include/llvm/CodeGenData/CodeGenDataWriter.h
@@ -0,0 +1,68 @@
+//===- CodeGenDataWriter.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for writing codegen data.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGENDATA_CODEGENDATAWRITER_H
+#define LLVM_CODEGENDATA_CODEGENDATAWRITER_H
+
+#include "llvm/CodeGenData/CodeGenData.h"
+#include "llvm/CodeGenData/OutlinedHashTreeRecord.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+
+class CGDataOStream;
+
+class CodeGenDataWriter {
+ /// The outlined hash tree to be written.
+ OutlinedHashTreeRecord HashTreeRecord;
+
+ /// A bit mask describing the kind of the codegen data.
+ CGDataKind DataKind = CGDataKind::Unknown;
+
+public:
+ CodeGenDataWriter() = default;
+ ~CodeGenDataWriter() = default;
+
+ /// Add the outlined hash tree record. The input Record is released.
+ void addRecord(OutlinedHashTreeRecord &Record);
+
+ /// Write the codegen data to \c OS
+ Error write(raw_fd_ostream &OS);
+
+ /// Write the codegen data in text format to \c OS
+ Error writeText(raw_fd_ostream &OS);
+
+ /// Return the attributes of the current CGData.
+ CGDataKind getCGDataKind() const { return DataKind; }
+
+ /// Return true if the header indicates the data has an outlined hash tree.
+ bool hasOutlinedHashTree() const {
+ return static_cast<uint32_t>(DataKind) &
+ static_cast<uint32_t>(CGDataKind::FunctionOutlinedHashTree);
+ }
+
+private:
+ /// The offset of the outlined hash tree in the file.
+ uint64_t OutlinedHashTreeOffset;
+
+ /// Write the codegen data header to \c COS
+ Error writeHeader(CGDataOStream &COS);
+
+ /// Write the codegen data header in text to \c OS
+ Error writeHeaderText(raw_fd_ostream &OS);
+
+ Error writeImpl(CGDataOStream &COS);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGENDATA_CODEGENDATAWRITER_H
diff --git a/llvm/lib/CodeGenData/CMakeLists.txt b/llvm/lib/CodeGenData/CMakeLists.txt
index 3ba90f96cc86d4..1156d53afb2e0f 100644
--- a/llvm/lib/CodeGenData/CMakeLists.txt
+++ b/llvm/lib/CodeGenData/CMakeLists.txt
@@ -1,4 +1,7 @@
add_llvm_component_library(LLVMCodeGenData
+ CodeGenData.cpp
+ CodeGenDataReader.cpp
+ CodeGenDataWriter.cpp
OutlinedHashTree.cpp
OutlinedHashTreeRecord.cpp
diff --git a/llvm/lib/CodeGenData/CodeGenData.cpp b/llvm/lib/CodeGenData/CodeGenData.cpp
new file mode 100644
index 00000000000000..3bd21c97c7de7a
--- /dev/null
+++ b/llvm/lib/CodeGenData/CodeGenData.cpp
@@ -0,0 +1,197 @@
+//===-- CodeGenData.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for codegen data that has stable summary which
+// can be used to optimize the code in the subsequent codegen.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Bitcode/BitcodeWriter.h"
+#include "llvm/CodeGenData/CodeGenDataReader.h"
+#include "llvm/CodeGenData/OutlinedHashTreeRecord.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/WithColor.h"
+
+#define DEBUG_TYPE "cg-data"
+
+using namespace llvm;
+using namespace cgdata;
+
+static std::string getCGDataErrString(cgdata_error Err,
+ const std::string &ErrMsg = "") {
+ std::string Msg;
+ raw_string_ostream OS(Msg);
+
+ switch (Err) {
+ case cgdata_error::success:
+ OS << "success";
+ break;
+ case cgdata_error::eof:
+ OS << "end of File";
+ break;
+ case cgdata_error::bad_magic:
+ OS << "invalid codegen data (bad magic)";
+ break;
+ case cgdata_error::bad_header:
+ OS << "invalid codegen data (file header is corrupt)";
+ break;
+ case cgdata_error::empty_cgdata:
+ OS << "empty codegen data";
+ break;
+ case cgdata_error::malformed:
+ OS << "malformed codegen data";
+ break;
+ case cgdata_error::unsupported_version:
+ OS << "unsupported codegen data version";
+ break;
+ }
+
+ // If optional error message is not empty, append it to the message.
+ if (!ErrMsg.empty())
+ OS << ": " << ErrMsg;
+
+ return OS.str();
+}
+
+namespace {
+
+// FIXME: This class is only here to support the transition to llvm::Error. It
+// will be removed once this transition is complete. Clients should prefer to
+// deal with the Error value directly, rather than converting to error_code.
+class CGDataErrorCategoryType : public std::error_category {
+ const char *name() const noexcept override { return "llvm.cgdata"; }
+
+ std::string message(int IE) const override {
+ return getCGDataErrString(static_cast<cgdata_error>(IE));
+ }
+};
+
+} // end anonymous namespace
+
+const std::error_category &llvm::cgdata_category() {
+ static CGDataErrorCategoryType ErrorCategory;
+ return ErrorCategory;
+}
+
+std::string CGDataError::message() const {
+ return getCGDataErrString(Err, Msg);
+}
+
+char CGDataError::ID = 0;
+
+namespace {
+
+const char *CodeGenDataSectNameCommon[] = {
+#define CG_DATA_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
+ SectNameCommon,
+#include "llvm/CodeGenData/CodeGenData.inc"
+};
+
+const char *CodeGenDataSectNameCoff[] = {
+#define CG_DATA_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) \
+ SectNameCoff,
+#include "llvm/CodeGenData/CodeGenData.inc"
+};
+
+const char *CodeGenDataSectNamePrefix[] = {
+#define CG_DATA_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Prefix,
+#include "llvm/CodeGenData/CodeGenData.inc"
+};
+
+} // namespace
+
+namespace llvm {
+
+std::string getCodeGenDataSectionName(CGDataSectKind CGSK,
+ Triple::ObjectFormatType OF,
+ bool AddSegmentInfo) {
+ std::string SectName;
+
+ if (OF == Triple::MachO && AddSegmentInfo)
+ SectName = CodeGenDataSectNamePrefix[CGSK];
+
+ if (OF == Triple::COFF)
+ SectName += CodeGenDataSectNameCoff[CGSK];
+ else
+ SectName += CodeGenDataSectNameCommon[CGSK];
+
+ return SectName;
+}
+
+std::unique_ptr<CodeGenData> CodeGenData::Instance = nullptr;
+std::once_flag CodeGenData::OnceFlag;
+
+CodeGenData &CodeGenData::getInstance() {
+ std::call_once(CodeGenData::OnceFlag, []() {
+ auto *CGD = new CodeGenData();
+ Instance.reset(CGD);
+
+ // TODO: Initialize writer or reader mode for the client optimization.
+ });
+ return *(Instance.get());
+}
+
+namespace IndexedCGData {
+
+Expected<Header> Header::readFromBuffer(const unsigned char *Curr) {
+ using namespace support;
+
+ static_assert(std::is_standard_layout_v<llvm::IndexedCGData::Header>,
+ "The header should be standard layout type since we use offset "
+ "of fields to read.");
+ Header H;
+ H.Magic = endian::readNext<uint64_t, endianness::little, unaligned>(Curr);
+ if (H.Magic != IndexedCGData::Magic)
+ return make_error<CGDataError>(cgdata_error::bad_magic);
+ H.Version = endian::readNext<uint32_t, endianness::little, unaligned>(Curr);
+ if (H.Version > IndexedCGData::CGDataVersion::CurrentVersion)
+ return make_error<CGDataError>(cgdata_error::unsupported_version);
+ H.DataKind = endian::readNext<uint32_t, endianness::little, unaligned>(Curr);
+
+ switch (H.Version) {
+ // When a new field is added to the header add a case statement here to
+ // compute the size as offset of the new field + size of the new field. This
+ // relies on the field being added to the end of the list.
+ static_assert(IndexedCGData::CGDataVersion::CurrentVersion == Version1,
+ "Please update the size computation below if a new field has "
+ "been added to the header, if not add a case statement to "
+ "fall through to the latest version.");
+ case 1ull:
+ H.OutlinedHashTreeOffset =
+ endian::readNext<uint64_t, endianness::little, unaligned>(Curr);
+ }
+
+ return H;
+}
+
+} // end namespace IndexedCGData
+
+namespace cgdata {
+
+void warn(Twine Message, std::string Whence, std::string Hint) {
+ WithColor::warning();
+ if (!Whence.empty())
+ errs() << Whence << ": ";
+ errs() << Message << "\n";
+ if (!Hint.empty())
+ WithColor::note() << Hint << "\n";
+}
+
+void warn(Error E, StringRef Whence) {
+ if (E.isA<CGDataError>()) {
+ handleAllErrors(std::move(E), [&](const CGDataError &IPE) {
+ warn(IPE.message(), std::string(Whence), std::string(""));
+ });
+ }
+}
+
+} // end namespace cgdata
+
+} // end namespace llvm
diff --git a/llvm/lib/CodeGenData/CodeGenDataReader.cpp b/llvm/lib/CodeGenData/CodeGenDataReader.cpp
new file mode 100644
index 00000000000000..1b08085dec2f25
--- /dev/null
+++ b/llvm/lib/CodeGenData/CodeGenDataReader.cpp
@@ -0,0 +1,174 @@
+//===- CodeGenDataReader.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for reading codegen data.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGenData/CodeGenDataReader.h"
+#include "llvm/CodeGenData/OutlinedHashTreeRecord.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+#define DEBUG_TYPE "cg-data-reader"
+
+using namespace llvm;
+
+namespace llvm {
+
+static Expected<std::unique_ptr<MemoryBuffer>>
+setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) {
+ auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN()
+ : FS.getBufferForFile(Filename);
+ if (std::error_code EC = BufferOrErr.getError())
+ return errorCodeToError(EC);
+ return std::move(BufferOrErr.get());
+}
+
+Error CodeGenDataReader::mergeFromObjectFile(
+ const object::ObjectFile *Obj,
+ OutlinedHashTreeRecord &GlobalOutlineRecord) {
+ Triple TT = Obj->makeTriple();
+ auto CGOutLineName =
+ getCodeGenDataSectionName(CG_outline, TT.getObjectFormat(), false);
+
+ for (auto &Section : Obj->sections()) {
+ Expected<StringRef> NameOrErr = Section.getName();
+ if (!NameOrErr)
+ return NameOrErr.takeError();
+ Expected<StringRef> ContentsOrErr = Section.getContents();
+ if (!ContentsOrErr)
+ return ContentsOrErr.takeError();
+ auto *Data = reinterpret_cast<const unsigned char *>(ContentsOrErr->data());
+ auto *EndData = Data + ContentsOrErr->size();
+
+ if (*NameOrErr == CGOutLineName) {
+ // In case dealing with an executable that has concatenaed cgdata,
+ // we want to merge them into a single cgdata.
+ // Although it's not a typical workflow, we support this scenario.
+ while (Data != EndData) {
+ OutlinedHashTreeRecord LocalOutlineRecord;
+ LocalOutlineRecord.deserialize(Data);
+ GlobalOutlineRecord.merge(LocalOutlineRecord);
+ }
+ }
+ // TODO: Add support for other cgdata sections.
+ }
+
+ return Error::success();
+}
+
+Error IndexedCodeGenDataReader::read() {
+ using namespace support;
+
+ // The smallest header with the version 1 is 24 bytes
+ const unsigned MinHeaderSize = 24;
+ if (DataBuffer->getBufferSize() < MinHeaderSize)
+ return error(cgdata_error::bad_header);
+
+ auto *Start =
+ reinterpret_cast<const unsigned char *>(DataBuffer->getBufferStart());
+ auto *End =
+ reinterpret_cast<const unsigned char *>(DataBuffer->getBufferEnd());
+ auto HeaderOr = IndexedCGData::Header::readFromBuffer(Start);
+ if (!HeaderOr)
+ return HeaderOr.takeError();
+ Header = HeaderOr.get();
+
+ if (hasOutlinedHashTree()) {
+ const unsigned char *Ptr = Start + Header.OutlinedHashTreeOffset;
+ if (Ptr >= End)
+ return error(cgdata_error::eof);
+ HashTreeRecord.deserialize(Ptr);
+ }
+
+ return success();
+}
+
+Expected<std::unique_ptr<CodeGenDataReader>>
+CodeGenDataReader::create(const Twine &Path, vfs::FileSystem &FS) {
+ // Set up the buffer to read.
+ auto BufferOrError = setupMemoryBuffer(Path, FS);
+ if (Error E = BufferOrError.takeError())
+ return std::move(E);
+ return CodeGenDataReader::create(std::move(BufferOrError.get()));
+}
+
+Expected<std::unique_ptr<CodeGenDataReader>>
+CodeGenDataReader::create(std::unique_ptr<MemoryBuffer> Buffer) {
+ if (Buffer->getBufferSize() == 0)
+ return make_error<CGDataError>(cgdata_error::empty_cgdata);
+
+ std::unique_ptr<CodeGenDataReader> Reader;
+ // Create the reader.
+ if (IndexedCodeGenDataReader::hasFormat(*Buffer))
+ Reader.reset(new IndexedCodeGenDataReader(std::move(Buffer)));
+ else if (TextCodeGenDataReader::hasFormat(*Buffer))
+ Reader.reset(new TextCodeGenDataReader(std::move(Buffer)));
+ else
+ return make_error<CGDataError>(cgdata_error::malformed);
+
+ // Initialize the reader and return the result.
+ if (Error E = Reader->read())
+ return std::move(E);
+
+ return std::move(Reader);
+}
+
+bool IndexedCodeGenDataReader::hasFormat(const MemoryBuffer &DataBuffer) {
+ using namespace support;
+ if (DataBuffer.getBufferSize() < 8)
+ return false;
+
+ uint64_t Magic = endian::read<uint64_t, llvm::endianness::little, aligned>(
+ DataBuffer.getBufferStart());
+ // Verify that it's magical.
+ return Magic == IndexedCGData::Magic;
+}
+
+bool TextCodeGenDataReader::hasFormat(const MemoryBuffer &Buffer) {
+ // Verify that this really looks like plain ASCII text by checking a
+ // 'reasonable' number of characters (up to profile magic size).
+ size_t count = std::min(Buffer.getBufferSize(), sizeof(uint64_t));
+ StringRef buffer = Buffer.getBufferStart();
+ return count == 0 ||
+ std::all_of(buffer.begin(), buffer.begin() + count,
+ [](char c) { return isPrint(c) || isSpace(c); });
+}
+Error TextCodeGenDataReader::read() {
+ using namespace support;
+
+ // Parse the custom header line by line.
+ while (Line->starts_with(":")) {
+ StringRef Str = Line->substr(1);
+ if (Str.equals_insensitive("outlined_hash_tree"))
+ DataKind |= CGDataKind::FunctionOutlinedHashTree;
+ else
+ return error(cgdata_error::bad_header);
+ ++Line;
+ }
+
+ // We treat an empty header (that as a comment # only) as a valid header.
+ if (Line.is_at_eof()) {
+ if (DataKind != CGDataKind::Unknown)
+ return error(cgdata_error::bad_header);
+ return Error::success();
+ }
+
+ // The YAML docs follow after the header.
+ const char *Pos = (*Line).data();
+ size_t Size = reinterpret_cast<size_t>(DataBuffer->getBufferEnd()) -
+ reinterpret_cast<size_t>(Pos);
+ yaml::Input YOS(StringRef(Pos, Size));
+ if (hasOutlinedHashTree())
+ HashTreeRecord.deserializeYAML(YOS);
+
+ // TODO: Add more yaml cgdata in order
+
+ return Error::success();
+}
+} // end namespace llvm
diff --git a/llvm/lib/CodeGenData/CodeGenDataWriter.cpp b/llvm/lib/CodeGenData/CodeGenDataWriter.cpp
new file mode 100644
index 00000000000000..9aa0d86223f714
--- /dev/null
+++ b/llvm/lib/CodeGenData/CodeGenDataWriter.cpp
@@ -0,0 +1,162 @@
+//===- CodeGenDataWriter.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for writing codegen data.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGenData/CodeGenDataWriter.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/EndianStream.h"
+
+#define DEBUG_TYPE "cg-data-writer"
+
+using namespace llvm;
+
+namespace llvm {
+
+/// A struct to define how the data stream should be patched.
+struct CGDataPatchItem {
+ uint64_t Pos; // Where to patch.
+ uint64_t *D; // Pointer to an array of source data.
+ int N; // Number of elements in \c D array.
+};
+
+// A wrapper class to abstract writer stream with support of bytes
+// back patching.
+class CGDataOStream {
+public:
+ CGDataOStream(raw_fd_ostream &FD)
+ : IsFDOStream(true), OS(FD), LE(FD, llvm::endianness::little) {}
+ CGDataOStream(raw_string_ostream &STR)
+ : IsFDOStream(false), OS(STR), LE(STR, llvm::endianness::little) {}
+
+ uint64_t tell() { return OS.tell(); }
+ void write(uint64_t V) { LE.write<uint64_t>(V); }
+ void write32(uint32_t V) { LE.write<uint32_t>(V); }
+ void write8(uint8_t V) { LE.write<uint8_t>(V); }
+
+ // \c patch can only be called when all data is written and flushed.
+ // For raw_string_ostream, the patch is done on the target string
+ // directly and it won't be reflected in the stream's internal buffer.
+ void patch(ArrayRef<CGDataPatchItem> P) {
+ using namespace support;
+
+ if (IsFDOStream) {
+ raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
+ const uint64_t LastPos = FDOStream.tell();
+ for (const auto &K : P) {
+ FDOStream.seek(K.Pos);
+ for (int I = 0; I < K.N; I++)
+ write(K.D[I]);
+ }
+ // Reset the stream to the last position after patching so that users
+ // don't accidentally overwrite data. This makes it consistent with
+ // the string stream below which replaces the data directly.
+ FDOStream.seek(LastPos);
+ } else {
+ raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
+ std::string &Data = SOStream.str(); // with flush
+ for (const auto &K : P) {
+ for (int I = 0; I < K.N; I++) {
+ uint64_t Bytes =
+ endian::byte_swap<uint64_t, llvm::endianness::little>(K.D[I]);
+ Data.replace(K.Pos + I * sizeof(uint64_t), sizeof(uint64_t),
+ (const char *)&Bytes, sizeof(uint64_t));
+ }
+ }
+ }
+ }
+
+ // If \c OS is an instance of \c raw_fd_ostream, this field will be
+ // true. Otherwise, \c OS will be an raw_string_ostream.
+ bool IsFDOStream;
+ raw_ostream &OS;
+ support::endian::Writer LE;
+};
+
+} // end namespace llvm
+
+void CodeGenDataWriter::addRecord(OutlinedHashTreeRecord &Record) {
+ assert(Record.HashTree && "empty hash tree in the record");
+ HashTreeRecord.HashTree = std::move(Record.HashTree);
+
+ DataKind |= CGDataKind::FunctionOutlinedHashTree;
+}
+
+Error CodeGenDataWriter::write(raw_fd_ostream &OS) {
+ CGDataOStream COS(OS);
+ return writeImpl(COS);
+}
+
+Error CodeGenDataWriter::writeHeader(CGDataOStream &COS) {
+ using namespace support;
+ IndexedCGData::Header Header;
+ Header.Magic = IndexedCGData::Magic;
+ Header.Version = IndexedCGData::Version;
+
+ // Set the CGDataKind depending on the kind.
+ Header.DataKind = 0;
+ if (static_cast<bool>(DataKind & CGDataKind::FunctionOutlinedHashTree))
+ Header.DataKind |=
+ static_cast<uint32_t>(CGDataKind::FunctionOutlinedHashTree);
+
+ Header.OutlinedHashTreeOffset = 0;
+
+ // Only write out up to the CGDataKind. We need to remember the offest of the
+ // remaing fields to allow back patching later.
+ COS.write(Header.Magic);
+ COS.write32(Header.Version);
+ COS.write32(Header.DataKind);
+
+ // Save the location of Header.OutlinedHashTreeOffset field in \c COS.
+ OutlinedHashTreeOffset = COS.tell();
+
+ // Reserve the space for OutlinedHashTreeOffset field.
+ COS.write(0);
+
+ return Error::success();
+}
+
+Error CodeGenDataWriter::writeImpl(CGDataOStream &COS) {
+ if (Error E = writeHeader(COS))
+ return E;
+
+ uint64_t OutlinedHashTreeFieldStart = COS.tell();
+ if (hasOutlinedHashTree())
+ HashTreeRecord.serialize(COS.OS);
+
+ // Back patch the offsets.
+ CGDataPatchItem PatchItems[] = {
+ {OutlinedHashTreeOffset, &OutlinedHashTreeFieldStart, 1}};
+ COS.patch(PatchItems);
+
+ return Error::success();
+}
+
+Error CodeGenDataWriter::writeHeaderText(raw_fd_ostream &OS) {
+ if (hasOutlinedHashTree())
+ OS << "# Outlined stable hash tree\n:outlined_hash_tree\n";
+
+ // TODO: Add more data types in this header
+
+ return Error::success();
+}
+
+Error CodeGenDataWriter::writeText(raw_fd_ostream &OS) {
+ if (Error E = writeHeaderText(OS))
+ return E;
+
+ yaml::Output YOS(OS);
+ if (hasOutlinedHashTree())
+ HashTreeRecord.serializeYAML(YOS);
+
+ // TODO: Write more yaml cgdata in order
+
+ return Error::success();
+}
diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt
index 6127b76db06b7f..be777ce650e874 100644
--- a/llvm/test/CMakeLists.txt
+++ b/llvm/test/CMakeLists.txt
@@ -73,6 +73,7 @@ set(LLVM_TEST_DEPENDS
llvm-c-test
llvm-cat
llvm-cfi-verify
+ llvm-cgdata
llvm-config
llvm-cov
llvm-cvtres
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 4c05317036d1a3..cc7e9d535a9c33 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -180,6 +180,7 @@ def get_asan_rtlib():
"llvm-addr2line",
"llvm-bcanalyzer",
"llvm-bitcode-strip",
+ "llvm-cgdata",
"llvm-config",
"llvm-cov",
"llvm-cxxdump",
diff --git a/llvm/test/tools/llvm-cgdata/dump.test b/llvm/test/tools/llvm-cgdata/dump.test
new file mode 100644
index 00000000000000..ce2ad27a5ff81c
--- /dev/null
+++ b/llvm/test/tools/llvm-cgdata/dump.test
@@ -0,0 +1,30 @@
+# Test dump between the binary and text formats.
+
+RUN: split-file %s %t
+
+RUN: llvm-cgdata dump -binary %t/dump.cgtext -o %t/dump.cgdata
+RUN: llvm-cgdata dump -text %t/dump.cgdata -o %t/dump-round.cgtext
+RUN: llvm-cgdata dump -binary %t/dump-round.cgtext -o %t/dump-round.cgdata
+RUN: diff %t/dump.cgdata %t/dump-round.cgdata
+
+;--- dump.cgtext
+# Outlined stable hash tree
+:outlined_hash_tree
+---
+0:
+ Hash: 0x0
+ Terminals: 0
+ SuccessorIds: [ 1 ]
+1:
+ Hash: 0x1
+ Terminals: 0
+ SuccessorIds: [ 2, 3 ]
+2:
+ Hash: 0x3
+ Terminals: 5
+ SuccessorIds: [ ]
+3:
+ Hash: 0x2
+ Terminals: 4
+ SuccessorIds: [ ]
+...
diff --git a/llvm/test/tools/llvm-cgdata/empty.test b/llvm/test/tools/llvm-cgdata/empty.test
new file mode 100644
index 00000000000000..d5e201b9eec17f
--- /dev/null
+++ b/llvm/test/tools/llvm-cgdata/empty.test
@@ -0,0 +1,32 @@
+# Test for empty cgdata file, which is invalid.
+RUN: touch %t_emptyfile.cgtext
+RUN: not llvm-cgdata dump %t_emptyfile.cgtext -text -o - 2>&1 | FileCheck %s --check-prefix ERROR
+ERROR: {{.}}emptyfile.cgtext: empty codegen data
+
+# Test for empty header in the text format. It can be converted to a valid binary file.
+RUN: printf '#' > %t_emptyheader.cgtext
+RUN: llvm-cgdata dump %t_emptyheader.cgtext -binary -o %t_emptyheader.cgdata
+
+# Without any cgdata other than the header, no data shows by default.
+RUN: llvm-cgdata show %t_emptyheader.cgdata | FileCheck %s --allow-empty --check-prefix EMPTY
+EMPTY-NOT: any
+
+# The version number appears when asked, as it's in the header
+RUN: llvm-cgdata show --cgdata-version %t_emptyheader.cgdata | FileCheck %s --check-prefix VERSION
+VERSION: Version: {{.}}
+
+# When converting a binary file (w/ the header only) to a text file, it's an empty file as the text format does not have an explicit header.
+RUN: llvm-cgdata dump %t_emptyheader.cgdata -text -o - | FileCheck %s --allow-empty --check-prefix EMPTY
+
+# Synthesize a header only cgdata.
+# struct Header {
+# uint64_t Magic;
+# uint32_t Version;
+# uint32_t DataKind;
+# uint64_t OutlinedHashTreeOffset;
+# }
+RUN: printf '\xffcgdata\x81' > %t_header.cgdata
+RUN: printf '\x01\x00\x00\x00' >> %t_header.cgdata
+RUN: printf '\x00\x00\x00\x00' >> %t_header.cgdata
+RUN: printf '\x18\x00\x00\x00\x00\x00\x00\x00' >> %t_header.cgdata
+RUN: diff %t_header.cgdata %t_emptyheader.cgdata
diff --git a/llvm/test/tools/llvm-cgdata/error.test b/llvm/test/tools/llvm-cgdata/error.test
new file mode 100644
index 00000000000000..5e1b14de5e509d
--- /dev/null
+++ b/llvm/test/tools/llvm-cgdata/error.test
@@ -0,0 +1,38 @@
+# Test various error cases
+
+# Synthesize a header only cgdata.
+# struct Header {
+# uint64_t Magic;
+# uint32_t Version;
+# uint32_t DataKind;
+# uint64_t OutlinedHashTreeOffset;
+# }
+RUN: touch %t_empty.cgdata
+RUN: not llvm-cgdata show %t_empty.cgdata 2>&1 | FileCheck %s --check-prefix EMPTY
+EMPTY: {{.}}cgdata: empty codegen data
+
+# Not a magic.
+RUN: printf '\xff' > %t_malformed.cgdata
+RUN: not llvm-cgdata show %t_malformed.cgdata 2>&1 | FileCheck %s --check-prefix MALFORMED
+MALFORMED: {{.}}cgdata: malformed codegen data
+
+# The minimum header size is 24.
+RUN: printf '\xffcgdata\x81' > %t_corrupt.cgdata
+RUN: not llvm-cgdata show %t_corrupt.cgdata 2>&1 | FileCheck %s --check-prefix CORRUPT
+CORRUPT: {{.}}cgdata: invalid codegen data (file header is corrupt)
+
+# The current version 1 while the header says 2.
+RUN: printf '\xffcgdata\x81' > %t_version.cgdata
+RUN: printf '\x02\x00\x00\x00' >> %t_version.cgdata
+RUN: printf '\x00\x00\x00\x00' >> %t_version.cgdata
+RUN: printf '\x18\x00\x00\x00\x00\x00\x00\x00' >> %t_version.cgdata
+RUN: not llvm-cgdata show %t_version.cgdata 2>&1 | FileCheck %s --check-prefix BAD_VERSION
+BAD_VERSION: {{.}}cgdata: unsupported codegen data version
+
+# Header says an outlined hash tree, but the file ends after the header.
+RUN: printf '\xffcgdata\x81' > %t_eof.cgdata
+RUN: printf '\x01\x00\x00\x00' >> %t_eof.cgdata
+RUN: printf '\x01\x00\x00\x00' >> %t_eof.cgdata
+RUN: printf '\x18\x00\x00\x00\x00\x00\x00\x00' >> %t_eof.cgdata
+RUN: not llvm-cgdata show %t_eof.cgdata 2>&1 | FileCheck %s --check-prefix EOF
+EOF: {{.}}cgdata: end of File
diff --git a/llvm/test/tools/llvm-cgdata/merge-archive.test b/llvm/test/tools/llvm-cgdata/merge-archive.test
new file mode 100644
index 00000000000000..a27d6c2a16f4ab
--- /dev/null
+++ b/llvm/test/tools/llvm-cgdata/merge-archive.test
@@ -0,0 +1,75 @@
+# Merge an archive that has two object files having cgdata (__llvm_outline)
+
+RUN: split-file %s %t
+
+RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-1.ll -o %t/merge-1.o
+RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-2.ll -o %t/merge-2.o
+RUN: llvm-ar rcs %t/merge-archive.a %t/merge-1.o %t/merge-2.o
+RUN: llvm-cgdata merge %t/merge-archive.a -o %t/merge-archive.cgdata
+RUN: llvm-cgdata show %t/merge-archive.cgdata | FileCheck %s
+CHECK: Outlined hash tree:
+CHECK-NEXT: Total Node Count: 4
+CHECK-NEXT: Terminal Node Count: 2
+CHECK-NEXT: Depth: 2
+
+RUN: llvm-cgdata dump %t/merge-archive.cgdata | FileCheck %s --check-prefix TREE
+TREE: # Outlined stable hash tree
+TREE-NEXT: :outlined_hash_tree
+TREE-NEXT: ---
+TREE-NEXT: 0:
+TREE-NEXT: Hash: 0x0
+TREE-NEXT: Terminals: 0
+TREE-NEXT: SuccessorIds: [ 1 ]
+TREE-NEXT: 1:
+TREE-NEXT: Hash: 0x1
+TREE-NEXT: Terminals: 0
+TREE-NEXT: SuccessorIds: [ 2, 3 ]
+TREE-NEXT: 2:
+TREE-NEXT: Hash: 0x3
+TREE-NEXT: Terminals: 5
+TREE-NEXT: SuccessorIds: [ ]
+TREE-NEXT: 3:
+TREE-NEXT: Hash: 0x2
+TREE-NEXT: Terminals: 4
+TREE-NEXT: SuccessorIds: [ ]
+TREE-NEXT: ...
+
+;--- merge-1.ll
+
+; The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp
+;---
+;0:
+; Hash: 0x0
+; Terminals: 0
+; SuccessorIds: [ 1 ]
+;1:
+; Hash: 0x1
+; Terminals: 0
+; SuccessorIds: [ 2 ]
+;2:
+; Hash: 0x2
+; Terminals: 4
+; SuccessorIds: [ ]
+;...
+
+ at .data = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\02\00\00\00\00\00\00\00\04\00\00\00\00\00\00\00", section "__DATA,__llvm_outline"
+
+;--- merge-2.ll
+
+; The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp
+;---
+;0:
+; Hash: 0x0
+; Terminals: 0
+; SuccessorIds: [ 1 ]
+;1:
+; Hash: 0x1
+; Terminals: 0
+; SuccessorIds: [ 2 ]
+;2:
+; Hash: 0x3
+; Terminals: 5
+; SuccessorIds: [ ]
+;...
+
+ at .data = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\03\00\00\00\00\00\00\00\05\00\00\00\00\00\00\00", section "__DATA,__llvm_outline"
diff --git a/llvm/test/tools/llvm-cgdata/merge-concat.test b/llvm/test/tools/llvm-cgdata/merge-concat.test
new file mode 100644
index 00000000000000..3411133cb7aacb
--- /dev/null
+++ b/llvm/test/tools/llvm-cgdata/merge-concat.test
@@ -0,0 +1,68 @@
+# Merge a binary file (e.g., a linked executable) having concatnated cgdata (__llvm_outline)
+
+RUN: split-file %s %t
+
+RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-concat.ll -o %t/merge-concat.o
+RUN: llvm-cgdata merge %t/merge-concat.o -o %t/merge-concat.cgdata
+RUN: llvm-cgdata show %t/merge-concat.cgdata | FileCheck %s
+CHECK: Outlined hash tree:
+CHECK-NEXT: Total Node Count: 4
+CHECK-NEXT: Terminal Node Count: 2
+CHECK-NEXT: Depth: 2
+
+RUN: llvm-cgdata dump %t/merge-concat.cgdata | FileCheck %s --check-prefix TREE
+TREE: # Outlined stable hash tree
+TREE-NEXT: :outlined_hash_tree
+TREE-NEXT: ---
+TREE-NEXT: 0:
+TREE-NEXT: Hash: 0x0
+TREE-NEXT: Terminals: 0
+TREE-NEXT: SuccessorIds: [ 1 ]
+TREE-NEXT: 1:
+TREE-NEXT: Hash: 0x1
+TREE-NEXT: Terminals: 0
+TREE-NEXT: SuccessorIds: [ 2, 3 ]
+TREE-NEXT: 2:
+TREE-NEXT: Hash: 0x3
+TREE-NEXT: Terminals: 5
+TREE-NEXT: SuccessorIds: [ ]
+TREE-NEXT: 3:
+TREE-NEXT: Hash: 0x2
+TREE-NEXT: Terminals: 4
+TREE-NEXT: SuccessorIds: [ ]
+TREE-NEXT: ...
+
+;--- merge-concat.ll
+
+; In an linked executable (as opposed to an object file), cgdata in __llvm_outline might be concatenated. Although this is not a typical workflow, we simply support this case to parse cgdata that is concatenated. In other word, the following two trees are encoded back-to-back in a binary format.
+;---
+;0:
+; Hash: 0x0
+; Terminals: 0
+; SuccessorIds: [ 1 ]
+;1:
+; Hash: 0x1
+; Terminals: 0
+; SuccessorIds: [ 2 ]
+;2:
+; Hash: 0x2
+; Terminals: 4
+; SuccessorIds: [ ]
+;...
+;---
+;0:
+; Hash: 0x0
+; Terminals: 0
+; SuccessorIds: [ 1 ]
+;1:
+; Hash: 0x1
+; Terminals: 0
+; SuccessorIds: [ 2 ]
+;2:
+; Hash: 0x3
+; Terminals: 5
+; SuccessorIds: [ ]
+;...
+
+ at .data1 = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\02\00\00\00\00\00\00\00\04\00\00\00\00\00\00\00", section "__DATA,__llvm_outline"
+ at .data2 = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\03\00\00\00\00\00\00\00\05\00\00\00\00\00\00\00", section "__DATA,__llvm_outline"
diff --git a/llvm/test/tools/llvm-cgdata/merge-double.test b/llvm/test/tools/llvm-cgdata/merge-double.test
new file mode 100644
index 00000000000000..6ce358cd72325b
--- /dev/null
+++ b/llvm/test/tools/llvm-cgdata/merge-double.test
@@ -0,0 +1,74 @@
+# Merge two object files having cgdata (__llvm_outline)
+
+RUN: split-file %s %t
+
+RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-1.ll -o %t/merge-1.o
+RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-2.ll -o %t/merge-2.o
+RUN: llvm-cgdata merge %t/merge-1.o %t/merge-2.o -o %t/merge.cgdata
+RUN: llvm-cgdata show %t/merge.cgdata | FileCheck %s
+CHECK: Outlined hash tree:
+CHECK-NEXT: Total Node Count: 4
+CHECK-NEXT: Terminal Node Count: 2
+CHECK-NEXT: Depth: 2
+
+RUN: llvm-cgdata dump %t/merge.cgdata | FileCheck %s --check-prefix TREE
+TREE: # Outlined stable hash tree
+TREE-NEXT: :outlined_hash_tree
+TREE-NEXT: ---
+TREE-NEXT: 0:
+TREE-NEXT: Hash: 0x0
+TREE-NEXT: Terminals: 0
+TREE-NEXT: SuccessorIds: [ 1 ]
+TREE-NEXT: 1:
+TREE-NEXT: Hash: 0x1
+TREE-NEXT: Terminals: 0
+TREE-NEXT: SuccessorIds: [ 2, 3 ]
+TREE-NEXT: 2:
+TREE-NEXT: Hash: 0x3
+TREE-NEXT: Terminals: 5
+TREE-NEXT: SuccessorIds: [ ]
+TREE-NEXT: 3:
+TREE-NEXT: Hash: 0x2
+TREE-NEXT: Terminals: 4
+TREE-NEXT: SuccessorIds: [ ]
+TREE-NEXT: ...
+
+;--- merge-1.ll
+
+; The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp
+;---
+;0:
+; Hash: 0x0
+; Terminals: 0
+; SuccessorIds: [ 1 ]
+;1:
+; Hash: 0x1
+; Terminals: 0
+; SuccessorIds: [ 2 ]
+;2:
+; Hash: 0x2
+; Terminals: 4
+; SuccessorIds: [ ]
+;...
+
+ at .data = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\02\00\00\00\00\00\00\00\04\00\00\00\00\00\00\00", section "__DATA,__llvm_outline"
+
+;--- merge-2.ll
+
+; The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp
+;---
+;0:
+; Hash: 0x0
+; Terminals: 0
+; SuccessorIds: [ 1 ]
+;1:
+; Hash: 0x1
+; Terminals: 0
+; SuccessorIds: [ 2 ]
+;2:
+; Hash: 0x3
+; Terminals: 5
+; SuccessorIds: [ ]
+;...
+
+ at .data = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\03\00\00\00\00\00\00\00\05\00\00\00\00\00\00\00", section "__DATA,__llvm_outline"
diff --git a/llvm/test/tools/llvm-cgdata/merge-single.test b/llvm/test/tools/llvm-cgdata/merge-single.test
new file mode 100644
index 00000000000000..73bdd9800dbe1d
--- /dev/null
+++ b/llvm/test/tools/llvm-cgdata/merge-single.test
@@ -0,0 +1,43 @@
+# Test merge a single object file into a cgdata
+
+RUN: split-file %s %t
+
+# Merge an object file that has no cgdata (__llvm_outline). It still produces a header only cgdata.
+RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-empty.ll -o %t/merge-empty.o
+RUN: llvm-cgdata merge %t/merge-empty.o -o %t/merge-empty.cgdata
+RUN: llvm-cgdata show %t/merge-empty.cgdata | FileCheck %s --allow-empty --check-prefix EMPTY
+EMPTY-NOT: any
+
+
+# Merge an object file having cgdata (__llvm_outline)
+RUN: llc -filetype=obj -mtriple arm64-apple-darwin %t/merge-single.ll -o %t/merge-single.o
+RUN: llvm-cgdata merge %t/merge-single.o -o %t/merge-single.cgdata
+RUN: llvm-cgdata show %t/merge-single.cgdata | FileCheck %s
+CHECK: Outlined hash tree:
+CHECK-NEXT: Total Node Count: 3
+CHECK-NEXT: Terminal Node Count: 1
+CHECK-NEXT: Depth: 2
+
+;--- merge-empty.ll
+ at .data = private unnamed_addr constant [1 x i8] c"\01"
+
+;--- merge-single.ll
+
+; The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp
+;---
+;0:
+; Hash: 0x0
+; Terminals: 0
+; SuccessorIds: [ 1 ]
+;1:
+; Hash: 0x1
+; Terminals: 0
+; SuccessorIds: [ 2 ]
+;2:
+; Hash: 0x2
+; Terminals: 4
+; SuccessorIds: [ ]
+;...
+
+ at .data = private unnamed_addr constant [72 x i8] c"\03\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\01\00\00\00\00\00\00\00\00\00\00\00\01\00\00\00\02\00\00\00\02\00\00\00\02\00\00\00\00\00\00\00\04\00\00\00\00\00\00\00", section "__DATA,__llvm_outline"
+
diff --git a/llvm/test/tools/llvm-cgdata/show.test b/llvm/test/tools/llvm-cgdata/show.test
new file mode 100644
index 00000000000000..accb4b77ede246
--- /dev/null
+++ b/llvm/test/tools/llvm-cgdata/show.test
@@ -0,0 +1,30 @@
+# Test show
+
+RUN: split-file %s %t
+RUN: llvm-cgdata show %t/show.cgtext | FileCheck %s
+
+CHECK: Outlined hash tree:
+CHECK-NEXT: Total Node Count: 3
+CHECK-NEXT: Terminal Node Count: 1
+CHECK-NEXT: Depth: 2
+
+# Convert the text file to the binary file
+RUN: llvm-cgdata dump -binary %t/show.cgtext -o %t/show.cgdata
+RUN: llvm-cgdata show %t/show.cgdata | FileCheck %s
+
+;--- show.cgtext
+:outlined_hash_tree
+---
+0:
+ Hash: 0x0
+ Terminals: 0
+ SuccessorIds: [ 1 ]
+1:
+ Hash: 0x1
+ Terminals: 0
+ SuccessorIds: [ 2 ]
+2:
+ Hash: 0x2
+ Terminals: 3
+ SuccessorIds: [ ]
+...
diff --git a/llvm/tools/llvm-cgdata/CMakeLists.txt b/llvm/tools/llvm-cgdata/CMakeLists.txt
new file mode 100644
index 00000000000000..4f1f7ff635bc3c
--- /dev/null
+++ b/llvm/tools/llvm-cgdata/CMakeLists.txt
@@ -0,0 +1,15 @@
+set(LLVM_LINK_COMPONENTS
+ CodeGen
+ CodeGenData
+ Core
+ Object
+ Support
+ )
+
+add_llvm_tool(llvm-cgdata
+ llvm-cgdata.cpp
+
+ DEPENDS
+ intrinsics_gen
+ GENERATE_DRIVER
+ )
diff --git a/llvm/tools/llvm-cgdata/llvm-cgdata.cpp b/llvm/tools/llvm-cgdata/llvm-cgdata.cpp
new file mode 100644
index 00000000000000..195f066fd6b872
--- /dev/null
+++ b/llvm/tools/llvm-cgdata/llvm-cgdata.cpp
@@ -0,0 +1,268 @@
+//===-- llvm-cgdata.cpp - LLVM CodeGen Data Tool --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// llvm-cgdata parses raw codegen data embedded in compiled binary files, and
+// merges them into a single .cgdata file. It can also inspect and maninuplate
+// a .cgdata file. This .cgdata can contain various codegen data like outlining
+// information, and it can be used to optimize the code in the subsequent build.
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/ADT/StringRef.h"
+#include "llvm/CodeGenData/CodeGenDataReader.h"
+#include "llvm/CodeGenData/CodeGenDataWriter.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/LLVMDriver.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/Support/WithColor.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+using namespace llvm::object;
+
+// TODO: https://llvm.org/docs/CommandGuide/llvm-cgdata.html has documentations
+// on each subcommand.
+cl::SubCommand DumpSubcommand(
+ "dump",
+ "Dump the (indexed) codegen data file in either text or binary format.");
+cl::SubCommand MergeSubcommand(
+ "merge", "Takes binary files having raw codegen data in custom sections, "
+ "and merge them into an index codegen data file.");
+cl::SubCommand
+ ShowSubcommand("show", "Show summary of the (indexed) codegen data file.");
+
+enum CGDataFormat {
+ CD_None = 0,
+ CD_Text,
+ CD_Binary,
+};
+
+cl::opt<std::string> OutputFilename("output", cl::value_desc("output"),
+ cl::init("-"), cl::desc("Output file"),
+ cl::sub(DumpSubcommand),
+ cl::sub(MergeSubcommand));
+cl::alias OutputFilenameA("o", cl::desc("Alias for --output"),
+ cl::aliasopt(OutputFilename));
+
+cl::opt<std::string> Filename(cl::Positional, cl::desc("<cgdata-file>"),
+ cl::sub(DumpSubcommand), cl::sub(ShowSubcommand));
+cl::list<std::string> InputFilenames(cl::Positional, cl::sub(MergeSubcommand),
+ cl::desc("<binary-files...>"));
+cl::opt<CGDataFormat> OutputFormat(
+ cl::desc("Format of output data"), cl::sub(DumpSubcommand),
+ cl::init(CD_Text),
+ cl::values(clEnumValN(CD_Text, "text", "Text encoding"),
+ clEnumValN(CD_Binary, "binary", "Binary encoding")));
+
+cl::opt<bool> ShowCGDataVersion("cgdata-version", cl::init(false),
+ cl::desc("Show cgdata version. "),
+ cl::sub(ShowSubcommand));
+
+static void exitWithError(Twine Message, std::string Whence = "",
+ std::string Hint = "") {
+ WithColor::error();
+ if (!Whence.empty())
+ errs() << Whence << ": ";
+ errs() << Message << "\n";
+ if (!Hint.empty())
+ WithColor::note() << Hint << "\n";
+ ::exit(1);
+}
+
+static void exitWithError(Error E, StringRef Whence = "") {
+ if (E.isA<CGDataError>()) {
+ handleAllErrors(std::move(E), [&](const CGDataError &IPE) {
+ exitWithError(IPE.message(), std::string(Whence));
+ });
+ return;
+ }
+
+ exitWithError(toString(std::move(E)), std::string(Whence));
+}
+
+static void exitWithErrorCode(std::error_code EC, StringRef Whence = "") {
+ exitWithError(EC.message(), std::string(Whence));
+}
+
+static int dump_main(int argc, const char *argv[]) {
+ if (Filename == OutputFilename) {
+ errs() << sys::path::filename(argv[0]) << " " << argv[1]
+ << ": Input file name cannot be the same as the output file name!\n";
+ return 1;
+ }
+
+ std::error_code EC;
+ raw_fd_ostream OS(OutputFilename.data(), EC,
+ OutputFormat == CD_Text ? sys::fs::OF_TextWithCRLF
+ : sys::fs::OF_None);
+ if (EC)
+ exitWithErrorCode(EC, OutputFilename);
+
+ auto FS = vfs::getRealFileSystem();
+ auto ReaderOrErr = CodeGenDataReader::create(Filename, *FS);
+ if (Error E = ReaderOrErr.takeError())
+ exitWithError(std::move(E), Filename);
+
+ CodeGenDataWriter Writer;
+ auto Reader = ReaderOrErr->get();
+ if (Reader->hasOutlinedHashTree()) {
+ OutlinedHashTreeRecord Record(Reader->releaseOutlinedHashTree());
+ Writer.addRecord(Record);
+ }
+
+ if (OutputFormat == CD_Text) {
+ if (Error E = Writer.writeText(OS))
+ exitWithError(std::move(E));
+ } else {
+ if (Error E = Writer.write(OS))
+ exitWithError(std::move(E));
+ }
+
+ return 0;
+}
+
+static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer,
+ OutlinedHashTreeRecord &GlobalOutlineRecord);
+
+static bool handleArchive(StringRef Filename, Archive &Arch,
+ OutlinedHashTreeRecord &GlobalOutlineRecord) {
+ bool Result = true;
+ Error Err = Error::success();
+ for (const auto &Child : Arch.children(Err)) {
+ auto BuffOrErr = Child.getMemoryBufferRef();
+ if (Error E = BuffOrErr.takeError())
+ exitWithError(std::move(E), Filename);
+ auto NameOrErr = Child.getName();
+ if (Error E = NameOrErr.takeError())
+ exitWithError(std::move(E), Filename);
+ std::string Name = (Filename + "(" + NameOrErr.get() + ")").str();
+ Result &= handleBuffer(Name, BuffOrErr.get(), GlobalOutlineRecord);
+ }
+ if (Err)
+ exitWithError(std::move(Err), Filename);
+ return Result;
+}
+
+static bool handleBuffer(StringRef Filename, MemoryBufferRef Buffer,
+ OutlinedHashTreeRecord &GlobalOutlineRecord) {
+ Expected<std::unique_ptr<Binary>> BinOrErr = object::createBinary(Buffer);
+ if (Error E = BinOrErr.takeError())
+ exitWithError(std::move(E), Filename);
+
+ bool Result = true;
+ if (auto *Obj = dyn_cast<ObjectFile>(BinOrErr->get())) {
+ if (Error E =
+ CodeGenDataReader::mergeFromObjectFile(Obj, GlobalOutlineRecord))
+ exitWithError(std::move(E), Filename);
+ } else if (auto *Arch = dyn_cast<Archive>(BinOrErr->get())) {
+ Result &= handleArchive(Filename, *Arch, GlobalOutlineRecord);
+ } else {
+ // TODO: Support for the MachO universal binary format.
+ errs() << "Error: unsupported binary file: " << Filename << "\n";
+ Result = false;
+ }
+
+ return Result;
+}
+
+static bool handleFile(StringRef Filename,
+ OutlinedHashTreeRecord &GlobalOutlineRecord) {
+ ErrorOr<std::unique_ptr<MemoryBuffer>> BuffOrErr =
+ MemoryBuffer::getFileOrSTDIN(Filename);
+ if (std::error_code EC = BuffOrErr.getError())
+ exitWithErrorCode(EC, Filename);
+ return handleBuffer(Filename, *BuffOrErr.get(), GlobalOutlineRecord);
+}
+
+static int merge_main(int argc, const char *argv[]) {
+ bool Result = true;
+ OutlinedHashTreeRecord GlobalOutlineRecord;
+ for (auto &Filename : InputFilenames)
+ Result &= handleFile(Filename, GlobalOutlineRecord);
+
+ if (!Result) {
+ errs() << "Error: failed to merge codegen data files.\n";
+ return 1;
+ }
+
+ CodeGenDataWriter Writer;
+ if (!GlobalOutlineRecord.empty())
+ Writer.addRecord(GlobalOutlineRecord);
+
+ std::error_code EC;
+ raw_fd_ostream Output(OutputFilename, EC, sys::fs::OF_None);
+ if (EC)
+ exitWithErrorCode(EC, OutputFilename);
+
+ if (auto E = Writer.write(Output))
+ exitWithError(std::move(E));
+
+ return 0;
+}
+
+static int show_main(int argc, const char *argv[]) {
+ if (Filename == OutputFilename) {
+ errs() << sys::path::filename(argv[0]) << " " << argv[1]
+ << ": Input file name cannot be the same as the output file name!\n";
+ return 1;
+ }
+
+ std::error_code EC;
+ raw_fd_ostream OS(OutputFilename.data(), EC, sys::fs::OF_TextWithCRLF);
+ if (EC)
+ exitWithErrorCode(EC, OutputFilename);
+
+ auto FS = vfs::getRealFileSystem();
+ auto ReaderOrErr = CodeGenDataReader::create(Filename, *FS);
+ if (Error E = ReaderOrErr.takeError())
+ exitWithError(std::move(E), Filename);
+
+ auto Reader = ReaderOrErr->get();
+ if (ShowCGDataVersion)
+ OS << "Version: " << Reader->getVersion() << "\n";
+
+ if (Reader->hasOutlinedHashTree()) {
+ auto Tree = Reader->releaseOutlinedHashTree();
+ OS << "Outlined hash tree:\n";
+ OS << " Total Node Count: " << Tree->size() << "\n";
+ OS << " Terminal Node Count: " << Tree->size(/*GetTerminalCountOnly=*/true)
+ << "\n";
+ OS << " Depth: " << Tree->depth() << "\n";
+ }
+
+ return 0;
+}
+
+int llvm_cgdata_main(int argc, char **argvNonConst, const llvm::ToolContext &) {
+ const char **argv = const_cast<const char **>(argvNonConst);
+
+ StringRef ProgName(sys::path::filename(argv[0]));
+
+ if (argc < 2) {
+ errs() << ProgName
+ << ": No subcommand specified! Run llvm-cgdata --help for usage.\n";
+ return 1;
+ }
+
+ cl::ParseCommandLineOptions(argc, argv, "LLVM codegen data\n");
+
+ if (DumpSubcommand)
+ return dump_main(argc, argv);
+
+ if (MergeSubcommand)
+ return merge_main(argc, argv);
+
+ if (ShowSubcommand)
+ return show_main(argc, argv);
+
+ errs() << ProgName
+ << ": Unknown command. Run llvm-cgdata --help for usage.\n";
+ return 1;
+}
>From 7faa26485bb2957e2eebe6073852c169e7292db1 Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee at meta.com>
Date: Wed, 24 Apr 2024 09:40:34 -0700
Subject: [PATCH 6/9] [MachineOutliner][NFC] Refactor
---
llvm/include/llvm/CodeGen/MachineOutliner.h | 5 +-
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 11 +++-
llvm/lib/CodeGen/MachineOutliner.cpp | 55 +++++++++++---------
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 7 +--
llvm/lib/Target/AArch64/AArch64InstrInfo.h | 3 +-
5 files changed, 47 insertions(+), 34 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineOutliner.h b/llvm/include/llvm/CodeGen/MachineOutliner.h
index eaba6c9b18f2bb..84937a8b563ac0 100644
--- a/llvm/include/llvm/CodeGen/MachineOutliner.h
+++ b/llvm/include/llvm/CodeGen/MachineOutliner.h
@@ -234,11 +234,11 @@ struct OutlinedFunction {
unsigned FrameConstructionID = 0;
/// Return the number of candidates for this \p OutlinedFunction.
- unsigned getOccurrenceCount() const { return Candidates.size(); }
+ virtual unsigned getOccurrenceCount() const { return Candidates.size(); }
/// Return the number of bytes it would take to outline this
/// function.
- unsigned getOutliningCost() const {
+ virtual unsigned getOutliningCost() const {
unsigned CallOverhead = 0;
for (const Candidate &C : Candidates)
CallOverhead += C.getCallOverhead();
@@ -272,6 +272,7 @@ struct OutlinedFunction {
}
OutlinedFunction() = delete;
+ virtual ~OutlinedFunction() = default;
};
} // namespace outliner
} // namespace llvm
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index d4a83e3753d980..1e7be312851929 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -2053,13 +2053,20 @@ class TargetInstrInfo : public MCInstrInfo {
/// Returns a \p outliner::OutlinedFunction struct containing target-specific
/// information for a set of outlining candidates. Returns std::nullopt if the
- /// candidates are not suitable for outlining.
+ /// candidates are not suitable for outlining. \p MinRep is the minimum
+ /// number of times the instruction sequence must be repeated.
virtual std::optional<outliner::OutlinedFunction> getOutliningCandidateInfo(
- std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
+ std::vector<outliner::Candidate> &RepeatedSequenceLocs,
+ unsigned MipRep) const {
llvm_unreachable(
"Target didn't implement TargetInstrInfo::getOutliningCandidateInfo!");
}
+ virtual std::optional<outliner::OutlinedFunction> getOutliningCandidateInfo(
+ std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
+ return getOutliningCandidateInfo(RepeatedSequenceLocs, /*MipRep=*/2);
+ }
+
/// Optional target hook to create the LLVM IR attributes for the outlined
/// function. If overridden, the overriding function must call the default
/// implementation.
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index 01a9966adb097a..a581997ec6c4cb 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -452,8 +452,9 @@ struct MachineOutliner : public ModulePass {
/// \param Mapper Contains outlining mapping information.
/// \param[out] FunctionList Filled with a list of \p OutlinedFunctions
/// each type of candidate.
- void findCandidates(InstructionMapper &Mapper,
- std::vector<OutlinedFunction> &FunctionList);
+ void
+ findCandidates(InstructionMapper &Mapper,
+ std::vector<std::unique_ptr<OutlinedFunction>> &FunctionList);
/// Replace the sequences of instructions represented by \p OutlinedFunctions
/// with calls to functions.
@@ -461,7 +462,9 @@ struct MachineOutliner : public ModulePass {
/// \param M The module we are outlining from.
/// \param FunctionList A list of functions to be inserted into the module.
/// \param Mapper Contains the instruction mappings for the module.
- bool outline(Module &M, std::vector<OutlinedFunction> &FunctionList,
+ /// \param[out] OutlinedFunctionNum The outlined function number.
+ bool outline(Module &M,
+ std::vector<std::unique_ptr<OutlinedFunction>> &FunctionList,
InstructionMapper &Mapper, unsigned &OutlinedFunctionNum);
/// Creates a function for \p OF and inserts it into the module.
@@ -580,7 +583,8 @@ void MachineOutliner::emitOutlinedFunctionRemark(OutlinedFunction &OF) {
}
void MachineOutliner::findCandidates(
- InstructionMapper &Mapper, std::vector<OutlinedFunction> &FunctionList) {
+ InstructionMapper &Mapper,
+ std::vector<std::unique_ptr<OutlinedFunction>> &FunctionList) {
FunctionList.clear();
SuffixTree ST(Mapper.UnsignedVec, OutlinerLeafDescendants);
@@ -682,7 +686,7 @@ void MachineOutliner::findCandidates(
continue;
}
- FunctionList.push_back(*OF);
+ FunctionList.push_back(std::make_unique<OutlinedFunction>(*OF));
}
}
@@ -827,10 +831,9 @@ MachineFunction *MachineOutliner::createOutlinedFunction(
return &MF;
}
-bool MachineOutliner::outline(Module &M,
- std::vector<OutlinedFunction> &FunctionList,
- InstructionMapper &Mapper,
- unsigned &OutlinedFunctionNum) {
+bool MachineOutliner::outline(
+ Module &M, std::vector<std::unique_ptr<OutlinedFunction>> &FunctionList,
+ InstructionMapper &Mapper, unsigned &OutlinedFunctionNum) {
LLVM_DEBUG(dbgs() << "*** Outlining ***\n");
LLVM_DEBUG(dbgs() << "NUMBER OF POTENTIAL FUNCTIONS: " << FunctionList.size()
<< "\n");
@@ -838,23 +841,23 @@ bool MachineOutliner::outline(Module &M,
// Sort by priority where priority := getNotOutlinedCost / getOutliningCost.
// The function with highest priority should be outlined first.
- stable_sort(FunctionList,
- [](const OutlinedFunction &LHS, const OutlinedFunction &RHS) {
- return LHS.getNotOutlinedCost() * RHS.getOutliningCost() >
- RHS.getNotOutlinedCost() * LHS.getOutliningCost();
- });
+ stable_sort(FunctionList, [](const std::unique_ptr<OutlinedFunction> &LHS,
+ const std::unique_ptr<OutlinedFunction> &RHS) {
+ return LHS->getNotOutlinedCost() * RHS->getOutliningCost() >
+ RHS->getNotOutlinedCost() * LHS->getOutliningCost();
+ });
// Walk over each function, outlining them as we go along. Functions are
// outlined greedily, based off the sort above.
auto *UnsignedVecBegin = Mapper.UnsignedVec.begin();
LLVM_DEBUG(dbgs() << "WALKING FUNCTION LIST\n");
- for (OutlinedFunction &OF : FunctionList) {
+ for (auto &OF : FunctionList) {
#ifndef NDEBUG
- auto NumCandidatesBefore = OF.Candidates.size();
+ auto NumCandidatesBefore = OF->Candidates.size();
#endif
// If we outlined something that overlapped with a candidate in a previous
// step, then we can't outline from it.
- erase_if(OF.Candidates, [&UnsignedVecBegin](Candidate &C) {
+ erase_if(OF->Candidates, [&UnsignedVecBegin](Candidate &C) {
return std::any_of(UnsignedVecBegin + C.getStartIdx(),
UnsignedVecBegin + C.getEndIdx() + 1, [](unsigned I) {
return I == static_cast<unsigned>(-1);
@@ -862,36 +865,36 @@ bool MachineOutliner::outline(Module &M,
});
#ifndef NDEBUG
- auto NumCandidatesAfter = OF.Candidates.size();
+ auto NumCandidatesAfter = OF->Candidates.size();
LLVM_DEBUG(dbgs() << "PRUNED: " << NumCandidatesBefore - NumCandidatesAfter
<< "/" << NumCandidatesBefore << " candidates\n");
#endif
// If we made it unbeneficial to outline this function, skip it.
- if (OF.getBenefit() < OutlinerBenefitThreshold) {
- LLVM_DEBUG(dbgs() << "SKIP: Expected benefit (" << OF.getBenefit()
+ if (OF->getBenefit() < OutlinerBenefitThreshold) {
+ LLVM_DEBUG(dbgs() << "SKIP: Expected benefit (" << OF->getBenefit()
<< " B) < threshold (" << OutlinerBenefitThreshold
<< " B)\n");
continue;
}
- LLVM_DEBUG(dbgs() << "OUTLINE: Expected benefit (" << OF.getBenefit()
+ LLVM_DEBUG(dbgs() << "OUTLINE: Expected benefit (" << OF->getBenefit()
<< " B) > threshold (" << OutlinerBenefitThreshold
<< " B)\n");
// It's beneficial. Create the function and outline its sequence's
// occurrences.
- OF.MF = createOutlinedFunction(M, OF, Mapper, OutlinedFunctionNum);
- emitOutlinedFunctionRemark(OF);
+ OF->MF = createOutlinedFunction(M, *OF, Mapper, OutlinedFunctionNum);
+ emitOutlinedFunctionRemark(*OF);
FunctionsCreated++;
OutlinedFunctionNum++; // Created a function, move to the next name.
- MachineFunction *MF = OF.MF;
+ MachineFunction *MF = OF->MF;
const TargetSubtargetInfo &STI = MF->getSubtarget();
const TargetInstrInfo &TII = *STI.getInstrInfo();
// Replace occurrences of the sequence with calls to the new function.
LLVM_DEBUG(dbgs() << "CREATE OUTLINED CALLS\n");
- for (Candidate &C : OF.Candidates) {
+ for (Candidate &C : OF->Candidates) {
MachineBasicBlock &MBB = *C.getMBB();
MachineBasicBlock::iterator StartIt = C.begin();
MachineBasicBlock::iterator EndIt = std::prev(C.end());
@@ -1183,7 +1186,7 @@ bool MachineOutliner::doOutline(Module &M, unsigned &OutlinedFunctionNum) {
// Prepare instruction mappings for the suffix tree.
populateMapper(Mapper, M, MMI);
- std::vector<OutlinedFunction> FunctionList;
+ std::vector<std::unique_ptr<OutlinedFunction>> FunctionList;
// Find all of the outlining candidates.
findCandidates(Mapper, FunctionList);
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 9518d573bccdd1..47783e11099688 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -8240,7 +8240,8 @@ static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
std::optional<outliner::OutlinedFunction>
AArch64InstrInfo::getOutliningCandidateInfo(
- std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
+ std::vector<outliner::Candidate> &RepeatedSequenceLocs,
+ unsigned MinRep) const {
outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
unsigned SequenceSize = 0;
@@ -8354,7 +8355,7 @@ AArch64InstrInfo::getOutliningCandidateInfo(
llvm::erase_if(RepeatedSequenceLocs, hasIllegalSPModification);
// If the sequence doesn't have enough candidates left, then we're done.
- if (RepeatedSequenceLocs.size() < 2)
+ if (RepeatedSequenceLocs.size() < MinRep)
return std::nullopt;
}
@@ -8598,7 +8599,7 @@ AArch64InstrInfo::getOutliningCandidateInfo(
}
// If we dropped all of the candidates, bail out here.
- if (RepeatedSequenceLocs.size() < 2) {
+ if (RepeatedSequenceLocs.size() < MinRep) {
RepeatedSequenceLocs.clear();
return std::nullopt;
}
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index 9a2914891675c5..f0eccd541c225a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -463,7 +463,8 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
bool isFunctionSafeToOutlineFrom(MachineFunction &MF,
bool OutlineFromLinkOnceODRs) const override;
std::optional<outliner::OutlinedFunction> getOutliningCandidateInfo(
- std::vector<outliner::Candidate> &RepeatedSequenceLocs) const override;
+ std::vector<outliner::Candidate> &RepeatedSequenceLocs,
+ unsigned MinRep) const override;
void mergeOutliningCandidateAttributes(
Function &F, std::vector<outliner::Candidate> &Candidates) const override;
outliner::InstrType
>From b5d7f5f918ff250763e916865e0ee5bc5c160474 Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee at meta.com>
Date: Wed, 24 Apr 2024 11:26:23 -0700
Subject: [PATCH 7/9] [CGData][MachineOutliner] Global Outlining
This commit introduces support for outlining functions across modules using codegen data generated from previous codegen. The codegen data currently manages the outlined hash tree, which records outlining instances that occurred locally in the past.
The machine outliner now operates in one of three modes:
1. CGDataMode::None: This is the default outliner mode that uses the suffix tree to identify (local) outlining candidates within a module. This mode is also used by (full)LTO to maintain optimal behavior with the combined module.
2. CGDataMode::Write (`codegen-data-generate`): This mode is identical to the default mode, but it also publishes the stable hash sequences of instructions in the outlined functions into a local outlined hash tree. It then encodes this into the `__llvm_outline` section, which will be dead-stripped at link time.
3. CGDataMode::Read (`codegen-data-use-path={.cgdata}`): This mode reads a codegen data file (.cgdata) and initializes a global outlined hash tree. This tree is used to generate global outlining candidates. Note that the codegen data file has been post-processed with the raw `__llvm_outline` sections from all native objects using the `llvm-cgdata` tool (or a linker, `LLD`, or a new ThinLTO pipeline later).
---
llvm/include/llvm/CodeGen/MachineOutliner.h | 36 +++
llvm/lib/CodeGen/CMakeLists.txt | 1 +
llvm/lib/CodeGen/MachineOutliner.cpp | 263 +++++++++++++++++-
llvm/lib/CodeGen/MachineStableHash.cpp | 20 +-
llvm/lib/CodeGenData/CodeGenData.cpp | 26 +-
llvm/test/CodeGen/AArch64/O3-pipeline.ll | 1 +
.../CodeGen/AArch64/cgdata-global-hash.ll | 40 +++
.../CodeGen/AArch64/cgdata-outlined-name.ll | 41 +++
.../AArch64/cgdata-read-double-outline.ll | 57 ++++
.../AArch64/cgdata-read-lto-outline.ll | 94 +++++++
.../CodeGen/AArch64/cgdata-read-priority.ll | 68 +++++
.../AArch64/cgdata-read-single-outline.ll | 42 +++
.../CodeGen/AArch64/cgdata-write-outline.ll | 51 ++++
llvm/test/CodeGen/RISCV/O3-pipeline.ll | 1 +
14 files changed, 736 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/cgdata-global-hash.ll
create mode 100644 llvm/test/CodeGen/AArch64/cgdata-outlined-name.ll
create mode 100644 llvm/test/CodeGen/AArch64/cgdata-read-double-outline.ll
create mode 100644 llvm/test/CodeGen/AArch64/cgdata-read-lto-outline.ll
create mode 100644 llvm/test/CodeGen/AArch64/cgdata-read-priority.ll
create mode 100644 llvm/test/CodeGen/AArch64/cgdata-read-single-outline.ll
create mode 100644 llvm/test/CodeGen/AArch64/cgdata-write-outline.ll
diff --git a/llvm/include/llvm/CodeGen/MachineOutliner.h b/llvm/include/llvm/CodeGen/MachineOutliner.h
index 84937a8b563ac0..3bda86f399b7b5 100644
--- a/llvm/include/llvm/CodeGen/MachineOutliner.h
+++ b/llvm/include/llvm/CodeGen/MachineOutliner.h
@@ -18,6 +18,7 @@
#include "llvm/CodeGen/LiveRegUnits.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineStableHash.h"
#include <initializer_list>
namespace llvm {
@@ -274,6 +275,41 @@ struct OutlinedFunction {
OutlinedFunction() = delete;
virtual ~OutlinedFunction() = default;
};
+
+/// The information necessary to create an outlined function that is matched
+/// globally.
+struct GlobalOutlinedFunction : public OutlinedFunction {
+ GlobalOutlinedFunction(OutlinedFunction &OF, unsigned GlobalOccurrenceCount)
+ : OutlinedFunction(OF.Candidates, OF.SequenceSize, OF.FrameOverhead,
+ OF.FrameConstructionID),
+ GlobalOccurrenceCount(GlobalOccurrenceCount) {}
+
+ unsigned GlobalOccurrenceCount;
+
+ /// Return the number of times that appear globally.
+ /// Global outlining candidate is uniquely created per each match, but this
+ /// might be erased out when it's overlapped with the previous outlining
+ /// instance.
+ unsigned getOccurrenceCount() const override {
+ assert(Candidates.size() <= 1);
+ return Candidates.empty() ? 0 : GlobalOccurrenceCount;
+ }
+
+ /// Return the outlining cost using the global occurrence count
+ /// with the same cost as the first (unique) candidate.
+ unsigned getOutliningCost() const override {
+ assert(Candidates.size() <= 1);
+ unsigned CallOverhead =
+ Candidates.empty()
+ ? 0
+ : Candidates[0].getCallOverhead() * getOccurrenceCount();
+ return CallOverhead + SequenceSize + FrameOverhead;
+ }
+
+ GlobalOutlinedFunction() = delete;
+ ~GlobalOutlinedFunction() = default;
+};
+
} // namespace outliner
} // namespace llvm
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 2c24de60edd43e..145442f0ce3e2c 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -266,6 +266,7 @@ add_llvm_component_library(LLVMCodeGen
Analysis
BitReader
BitWriter
+ CodeGenData
CodeGenTypes
Core
MC
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index a581997ec6c4cb..bca5a5bf064493 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -59,6 +59,7 @@
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/ModuleSummaryAnalysis.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
@@ -66,6 +67,7 @@
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/CodeGenData/CodeGenDataReader.h"
#include "llvm/IR/DIBuilder.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Mangler.h"
@@ -74,6 +76,7 @@
#include "llvm/Support/Debug.h"
#include "llvm/Support/SuffixTree.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/ModuleUtils.h"
#include <functional>
#include <tuple>
#include <vector>
@@ -97,6 +100,10 @@ STATISTIC(NumInvisible,
"Invisible instructions skipped during mapping");
STATISTIC(UnsignedVecSize,
"Total number of instructions mapped and saved to mapping vector");
+STATISTIC(StableHashAttempts,
+ "Count of hashing attempts made for outlined functions");
+STATISTIC(StableHashDropped,
+ "Count of unsuccessful hashing attempts for outlined functions");
// Set to true if the user wants the outliner to run on linkonceodr linkage
// functions. This is false by default because the linker can dedupe linkonceodr
@@ -127,6 +134,19 @@ static cl::opt<bool> OutlinerLeafDescendants(
"tree as candidates for outlining (if false, only leaf children "
"are considered)"));
+static cl::opt<bool>
+ DisableGlobalOutlining("disable-global-outlining", cl::Hidden,
+ cl::desc("Disable global outlining only by ignoring "
+ "the codegen data generation or use"),
+ cl::init(false));
+
+static cl::opt<bool> AppendContentHashToOutlinedName(
+ "append-content-hash-outlined-name", cl::Hidden,
+ cl::desc("This appends the content hash to the globally outlined function "
+ "name. It's beneficial for enhancing the precision of the stable "
+ "hash and for ordering the outlined functions."),
+ cl::init(true));
+
namespace {
/// Maps \p MachineInstrs to unsigned integers and stores the mappings.
@@ -417,11 +437,32 @@ struct MachineOutliner : public ModulePass {
/// Set when the pass is constructed in TargetPassConfig.
bool RunOnAllFunctions = true;
+ /// This is a compact representation of hash sequences of outlined functions.
+ /// It is used when OutlinerMode = CGDataMode::Write.
+ /// The resulting hash tree will be emitted into __llvm_outlined section
+ /// which will be dead-stripped not going to the final binary.
+ /// A post-process using llvm-cgdata, lld, or ThinLTO can merge them into
+ /// a global oulined hash tree for the subsequent codegen.
+ std::unique_ptr<OutlinedHashTree> LocalHashTree;
+
+ /// The combined index to check a LTO mode.
+ const ModuleSummaryIndex *TheIndex = nullptr;
+
+ /// The mode of the outliner.
+ /// When is's CGDataMode::None, candidates are populated with the suffix tree
+ /// within a module and outlined.
+ /// When it's CGDataMode::Write, in addition to CGDataMode::None, the hash
+ /// sequences of outlined functions are published into LocalHashTree.
+ /// When it's CGDataMode::Read, candidates are populated with the global
+ /// outlined hash tree that has been built by the previous codegen.
+ CGDataMode OutlinerMode = CGDataMode::None;
+
StringRef getPassName() const override { return "Machine Outliner"; }
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<MachineModuleInfoWrapperPass>();
AU.addPreserved<MachineModuleInfoWrapperPass>();
+ AU.addRequired<ImmutableModuleSummaryIndexWrapperPass>();
AU.setPreservesAll();
ModulePass::getAnalysisUsage(AU);
}
@@ -456,6 +497,16 @@ struct MachineOutliner : public ModulePass {
findCandidates(InstructionMapper &Mapper,
std::vector<std::unique_ptr<OutlinedFunction>> &FunctionList);
+ /// Find all repeated substrings that match in the global outlined hash
+ /// tree built from the previous codegen.
+ ///
+ /// \param Mapper Contains outlining mapping information.
+ /// \param[out] FunctionList Filled with a list of \p OutlinedFunctions
+ /// each type of candidate.
+ void findGlobalCandidates(
+ InstructionMapper &Mapper,
+ std::vector<std::unique_ptr<OutlinedFunction>> &FunctionList);
+
/// Replace the sequences of instructions represented by \p OutlinedFunctions
/// with calls to functions.
///
@@ -472,6 +523,17 @@ struct MachineOutliner : public ModulePass {
InstructionMapper &Mapper,
unsigned Name);
+ /// Compute and publish the stable hash sequence of instructions in the
+ /// outlined function, \p MF. The parameter \p CandSize represents the number
+ /// of candidates that have identical instruction sequences to \p MF.
+ void computeAndPublishHashSequence(MachineFunction &MF, unsigned CandSize);
+
+ /// Initialize the outliner mode.
+ void initializeOutlinerMode(const Module &M);
+
+ /// Emit the outlined hash tree into __llvm_outline section.
+ void emitOutlinedHashTree(Module &M);
+
/// Calls 'doOutline()' 1 + OutlinerReruns times.
bool runOnModule(Module &M) override;
@@ -582,6 +644,109 @@ void MachineOutliner::emitOutlinedFunctionRemark(OutlinedFunction &OF) {
MORE.emit(R);
}
+struct MatchedEntry {
+ size_t StartIdx;
+ size_t Length;
+ size_t Count;
+};
+
+static const HashNode *followHashNode(stable_hash StableHash,
+ const HashNode *Current) {
+ auto I = Current->Successors.find(StableHash);
+ return (I == Current->Successors.end()) ? nullptr : I->second.get();
+}
+
+// Find all matches in the global outlined hash tree.
+// It's quadratic complexity in theory, but it's nearly linear in practice
+// since the length of outlined sequences are small within a block.
+static std::vector<MatchedEntry> getMatchedEntries(InstructionMapper &Mapper) {
+ auto &InstrList = Mapper.InstrList;
+ auto &UnsignedVec = Mapper.UnsignedVec;
+
+ std::vector<MatchedEntry> MatchedEntries;
+ std::vector<stable_hash> Sequence;
+ auto Size = UnsignedVec.size();
+
+ // Get the global outlined hash tree built from the previous run.
+ assert(cgdata::hasOutlinedHashTree());
+ const auto *RootNode = cgdata::getOutlinedHashTree()->getRoot();
+ for (size_t I = 0; I < Size; ++I) {
+ // skip the invalid mapping that represents a large negative value.
+ if (UnsignedVec[I] >= Size)
+ continue;
+ const MachineInstr &MI = *InstrList[I];
+ // skip debug instructions as we did for the outlined function.
+ if (MI.isDebugInstr())
+ continue;
+ // skip the empty hash value.
+ stable_hash StableHashI = stableHashValue(MI);
+ if (!StableHashI)
+ continue;
+ Sequence.clear();
+ Sequence.push_back(StableHashI);
+
+ const HashNode *LastNode = followHashNode(StableHashI, RootNode);
+ if (!LastNode)
+ continue;
+
+ size_t J = I + 1;
+ for (; J < Size; ++J) {
+ // break on the invalid mapping that represents a large negative value.
+ if (UnsignedVec[J] >= Size)
+ break;
+ // ignore debug instructions as we did for the outlined function.
+ const MachineInstr &MJ = *InstrList[J];
+ if (MJ.isDebugInstr())
+ continue;
+ // break on the empty hash value.
+ stable_hash StableHashJ = stableHashValue(MJ);
+ if (!StableHashJ)
+ break;
+ LastNode = followHashNode(StableHashJ, LastNode);
+ if (!LastNode)
+ break;
+
+ // Even with a match ending with a terminal, we continue finding
+ // matches to populate all candidates.
+ Sequence.push_back(StableHashJ);
+ size_t Count = LastNode->Terminals;
+ if (Count)
+ MatchedEntries.push_back({I, J - I + 1, Count});
+ }
+ }
+
+ return MatchedEntries;
+}
+
+void MachineOutliner::findGlobalCandidates(
+ InstructionMapper &Mapper,
+ std::vector<std::unique_ptr<OutlinedFunction>> &FunctionList) {
+ FunctionList.clear();
+ auto &InstrList = Mapper.InstrList;
+ auto &MBBFlagsMap = Mapper.MBBFlagsMap;
+
+ std::vector<Candidate> CandidatesForRepeatedSeq;
+ for (auto &ME : getMatchedEntries(Mapper)) {
+ CandidatesForRepeatedSeq.clear();
+ MachineBasicBlock::iterator StartIt = InstrList[ME.StartIdx];
+ MachineBasicBlock::iterator EndIt = InstrList[ME.StartIdx + ME.Length - 1];
+ MachineBasicBlock *MBB = StartIt->getParent();
+ Candidate C(ME.StartIdx, ME.Length, StartIt, EndIt, MBB,
+ FunctionList.size(), MBBFlagsMap[MBB]);
+ CandidatesForRepeatedSeq.push_back(C);
+ const TargetInstrInfo *TII = C.getMF()->getSubtarget().getInstrInfo();
+ std::optional<OutlinedFunction> OF =
+ TII->getOutliningCandidateInfo(CandidatesForRepeatedSeq, /*MinRep*/ 1);
+ if (!OF || OF->Candidates.empty())
+ continue;
+ // We create a global candidate each match.
+ assert(OF->Candidates.size() == 1);
+
+ FunctionList.push_back(
+ std::make_unique<GlobalOutlinedFunction>(*OF, ME.Count));
+ }
+}
+
void MachineOutliner::findCandidates(
InstructionMapper &Mapper,
std::vector<std::unique_ptr<OutlinedFunction>> &FunctionList) {
@@ -690,6 +855,40 @@ void MachineOutliner::findCandidates(
}
}
+void MachineOutliner::computeAndPublishHashSequence(MachineFunction &MF,
+ unsigned CandSize) {
+ // Compute the hash sequence for the outlined function.
+ std::vector<stable_hash> OutlinedHashSequence;
+ for (auto &MBB : MF) {
+ for (auto &NewMI : MBB) {
+ stable_hash Hash = stableHashValue(NewMI);
+ if (!Hash) {
+ OutlinedHashSequence.clear();
+ break;
+ }
+ OutlinedHashSequence.push_back(Hash);
+ }
+ }
+
+ // Append a unique name based on the non-empty hash sequence.
+ if (AppendContentHashToOutlinedName && !OutlinedHashSequence.empty()) {
+ auto CombinedHash = stable_hash_combine_range(OutlinedHashSequence.begin(),
+ OutlinedHashSequence.end());
+ auto NewName =
+ MF.getName().str() + ".content." + std::to_string(CombinedHash);
+ MF.getFunction().setName(NewName);
+ }
+
+ // Publish the non-empty hash sequence to the local hash tree.
+ if (OutlinerMode == CGDataMode::Write) {
+ StableHashAttempts++;
+ if (!OutlinedHashSequence.empty())
+ LocalHashTree->insert({OutlinedHashSequence, CandSize});
+ else
+ StableHashDropped++;
+ }
+}
+
MachineFunction *MachineOutliner::createOutlinedFunction(
Module &M, OutlinedFunction &OF, InstructionMapper &Mapper, unsigned Name) {
@@ -766,6 +965,9 @@ MachineFunction *MachineOutliner::createOutlinedFunction(
}
}
+ if (OutlinerMode != CGDataMode::None)
+ computeAndPublishHashSequence(MF, OF.Candidates.size());
+
// Set normal properties for a late MachineFunction.
MF.getProperties().reset(MachineFunctionProperties::Property::IsSSA);
MF.getProperties().set(MachineFunctionProperties::Property::NoPHIs);
@@ -1134,12 +1336,65 @@ void MachineOutliner::emitInstrCountChangedRemark(
}
}
+void MachineOutliner::initializeOutlinerMode(const Module &M) {
+ if (DisableGlobalOutlining)
+ return;
+
+ if (auto *IndexWrapperPass =
+ getAnalysisIfAvailable<ImmutableModuleSummaryIndexWrapperPass>())
+ TheIndex = IndexWrapperPass->getIndex();
+
+ // (Full)LTO module does not have functions added to the index.
+ // In this case, we run the outliner without using codegen data as usual.
+ if (TheIndex && !TheIndex->hasExportedFunctions(M))
+ return;
+
+ // When codegen data write is enabled, we want to write the local outlined
+ // hash tree to the custom section, `__llvm_outline`.
+ // When the outlined hash tree is available from the previous codegen data,
+ // we want to read it to optimistically create global outlining candidates.
+ if (cgdata::emitCGData()) {
+ OutlinerMode = CGDataMode::Write;
+ // Create a local outlined hash tree to be published.
+ LocalHashTree.reset(new OutlinedHashTree());
+ // We don't need to read the outlined hash tree from the previous codegen
+ } else if (cgdata::hasOutlinedHashTree())
+ OutlinerMode = CGDataMode::Read;
+}
+
+void MachineOutliner::emitOutlinedHashTree(Module &M) {
+ assert(LocalHashTree);
+ if (!LocalHashTree->empty()) {
+ LLVM_DEBUG({
+ dbgs() << "Emit outlined hash tree. Size: " << LocalHashTree->size()
+ << "\n";
+ });
+ SmallVector<char> Buf;
+ raw_svector_ostream OS(Buf);
+
+ OutlinedHashTreeRecord HTR(std::move(LocalHashTree));
+ HTR.serialize(OS);
+
+ llvm::StringRef Data(Buf.data(), Buf.size());
+ std::unique_ptr<MemoryBuffer> Buffer =
+ MemoryBuffer::getMemBuffer(Data, "in-memory outlined hash tree", false);
+
+ Triple TT(M.getTargetTriple());
+ embedBufferInModule(
+ M, *Buffer.get(),
+ getCodeGenDataSectionName(CG_outline, TT.getObjectFormat()));
+ }
+}
+
bool MachineOutliner::runOnModule(Module &M) {
// Check if there's anything in the module. If it's empty, then there's
// nothing to outline.
if (M.empty())
return false;
+ // Initialize the outliner mode.
+ initializeOutlinerMode(M);
+
// Number to append to the current outlined function.
unsigned OutlinedFunctionNum = 0;
@@ -1159,6 +1414,9 @@ bool MachineOutliner::runOnModule(Module &M) {
}
}
+ if (OutlinerMode == CGDataMode::Write)
+ emitOutlinedHashTree(M);
+
return true;
}
@@ -1189,7 +1447,10 @@ bool MachineOutliner::doOutline(Module &M, unsigned &OutlinedFunctionNum) {
std::vector<std::unique_ptr<OutlinedFunction>> FunctionList;
// Find all of the outlining candidates.
- findCandidates(Mapper, FunctionList);
+ if (OutlinerMode == CGDataMode::Read)
+ findGlobalCandidates(Mapper, FunctionList);
+ else
+ findCandidates(Mapper, FunctionList);
// If we've requested size remarks, then collect the MI counts of every
// function before outlining, and the MI counts after outlining.
diff --git a/llvm/lib/CodeGen/MachineStableHash.cpp b/llvm/lib/CodeGen/MachineStableHash.cpp
index 5abfbd5981fba8..3e42a6f3ceccaf 100644
--- a/llvm/lib/CodeGen/MachineStableHash.cpp
+++ b/llvm/lib/CodeGen/MachineStableHash.cpp
@@ -94,9 +94,23 @@ stable_hash llvm::stableHashValue(const MachineOperand &MO) {
case MachineOperand::MO_Metadata:
StableHashBailingMetadataUnsupported++;
return 0;
- case MachineOperand::MO_GlobalAddress:
- StableHashBailingGlobalAddress++;
- return 0;
+ case MachineOperand::MO_GlobalAddress: {
+ const GlobalValue *GV = MO.getGlobal();
+ if (GV->hasPrivateLinkage() || !GV->hasName()) {
+ StableHashBailingGlobalAddress++;
+ return 0;
+ }
+ auto Name = GV->getName();
+ // Use the content hash of the outlined function.
+ auto Pos = Name.find_last_of(".content.");
+ if (Pos != StringRef::npos) {
+ assert(Name.starts_with("OUTLINED_FUNCTION"));
+ Name = Name.substr(Pos);
+ }
+ return stable_hash_combine(MO.getType(), MO.getTargetFlags(),
+ stable_hash_combine_string(Name),
+ MO.getOffset());
+ }
case MachineOperand::MO_TargetIndex: {
if (const char *Name = MO.getTargetIndexName())
return stable_hash_combine(MO.getType(), MO.getTargetFlags(),
diff --git a/llvm/lib/CodeGenData/CodeGenData.cpp b/llvm/lib/CodeGenData/CodeGenData.cpp
index 3bd21c97c7de7a..841af8a347eeb2 100644
--- a/llvm/lib/CodeGenData/CodeGenData.cpp
+++ b/llvm/lib/CodeGenData/CodeGenData.cpp
@@ -24,6 +24,13 @@
using namespace llvm;
using namespace cgdata;
+cl::opt<bool>
+ CodeGenDataGenerate("codegen-data-generate", cl::init(false), cl::Hidden,
+ cl::desc("Emit CodeGen Data into custom sections"));
+cl::opt<std::string>
+ CodeGenDataUsePath("codegen-data-use-path", cl::init(""), cl::Hidden,
+ cl::desc("File path to where .cgdata file is read"));
+
static std::string getCGDataErrString(cgdata_error Err,
const std::string &ErrMsg = "") {
std::string Msg;
@@ -133,7 +140,24 @@ CodeGenData &CodeGenData::getInstance() {
auto *CGD = new CodeGenData();
Instance.reset(CGD);
- // TODO: Initialize writer or reader mode for the client optimization.
+ if (CodeGenDataGenerate)
+ CGD->EmitCGData = true;
+ else if (!CodeGenDataUsePath.empty()) {
+ // Initialize the global CGData if the input file name is given.
+ // We do not error-out when failing to parse the input file.
+ // Instead, just emit an warning message and fall back as if no CGData
+ // were available.
+ auto FS = vfs::getRealFileSystem();
+ auto ReaderOrErr = CodeGenDataReader::create(CodeGenDataUsePath, *FS);
+ if (Error E = ReaderOrErr.takeError()) {
+ warn(std::move(E), CodeGenDataUsePath);
+ return;
+ }
+ // Publish each CGData based on the data type in the header.
+ auto Reader = ReaderOrErr->get();
+ if (Reader->hasOutlinedHashTree())
+ CGD->publishOutlinedHashTree(Reader->releaseOutlinedHashTree());
+ }
});
return *(Instance.get());
}
diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
index d3c8e3b7e805c1..391d63d2ceeaaf 100644
--- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
@@ -16,6 +16,7 @@
; CHECK-NEXT: Machine Branch Probability Analysis
; CHECK-NEXT: Default Regalloc Eviction Advisor
; CHECK-NEXT: Default Regalloc Priority Advisor
+; CHECK-NEXT: Module summary info
; CHECK-NEXT: ModulePass Manager
; CHECK-NEXT: Pre-ISel Intrinsic Lowering
; CHECK-NEXT: FunctionPass Manager
diff --git a/llvm/test/CodeGen/AArch64/cgdata-global-hash.ll b/llvm/test/CodeGen/AArch64/cgdata-global-hash.ll
new file mode 100644
index 00000000000000..09eb639ab40ef5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-global-hash.ll
@@ -0,0 +1,40 @@
+; This test verifies the stable hash values for different global variables
+; that have distinct names.
+; We generate two different cgdata files from nearly identical outline instances,
+; with the only difference being the last call target globals, @g vs @h.
+
+; RUN: split-file %s %t
+
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-generate=true -filetype=obj %t/local-g.ll -o %t/local-g.o
+; RUN: llvm-cgdata merge %t/local-g.o -o %t/local-g.cgdata
+; RUN: llvm-cgdata dump %t/local-g.cgdata -o %t/local-g.cgtext
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-generate=true -filetype=obj %t/local-h.ll -o %t/local-h.o
+; RUN: llvm-cgdata merge %t/local-h.o -o %t/local-h.cgdata
+; RUN: llvm-cgdata dump %t/local-h.cgdata -o %t/local-h.cgtext
+
+; We compare the trees which are only different at the terminal node's hash value.
+; Here we simply count the different lines that have `Hash` string.
+; RUN: not diff %t/local-g.cgtext %t/local-h.cgtext 2>&1 | grep Hash | wc -l | FileCheck %s
+; CHECK: 2
+
+;--- local-g.ll
+declare i32 @g(i32, i32, i32)
+define i32 @f1() minsize {
+ %1 = call i32 @g(i32 10, i32 1, i32 2);
+ ret i32 %1
+}
+define i32 @f2() minsize {
+ %1 = call i32 @g(i32 20, i32 1, i32 2);
+ ret i32 %1
+}
+
+;--- local-h.ll
+declare i32 @h(i32, i32, i32)
+define i32 @f1() minsize {
+ %1 = call i32 @h(i32 10, i32 1, i32 2);
+ ret i32 %1
+}
+define i32 @f2() minsize {
+ %1 = call i32 @h(i32 20, i32 1, i32 2);
+ ret i32 %1
+}
diff --git a/llvm/test/CodeGen/AArch64/cgdata-outlined-name.ll b/llvm/test/CodeGen/AArch64/cgdata-outlined-name.ll
new file mode 100644
index 00000000000000..fef4966afb2dba
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-outlined-name.ll
@@ -0,0 +1,41 @@
+; This test verifies the globally outlined function name has the content hash.
+
+; RUN: split-file %s %t
+
+; Check if the outlined function name has the content hash depending the flag.
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-generate=true -append-content-hash-outlined-name=false -filetype=obj %t/local-two.ll -o %t_write_base
+; RUN: llvm-objdump -d %t_write_base | FileCheck %s --check-prefix=BASE
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-generate=true -append-content-hash-outlined-name=true -filetype=obj %t/local-two.ll -o %t_write_suffix
+; RUN: llvm-objdump -d %t_write_suffix | FileCheck %s --check-prefix=SUFFIX
+; BASE-NOT: _OUTLINED_FUNCTION_{{.*}}.content.{{[0-9]+}}
+; SUFFIX: _OUTLINED_FUNCTION_{{.*}}.content.{{[0-9]+}}
+
+; Generate the cgdata file from each case and show they are identical.
+; RUN: llvm-cgdata merge %t_write_base -o %t_cgdata_base
+; RUN: llvm-cgdata merge %t_write_suffix -o %t_cgdata_suffix
+; RUN: diff %t_cgdata_base %t_cgdata_suffix
+
+; Read the cgdata in the machine outliner for optimistically outlining in local-one.ll.
+; Check if the outlined function has the content hash depending the flag.
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-use-path=%t_cgdata_base -append-content-hash-outlined-name=false -filetype=obj %t/local-one.ll -o %t_read_base
+; RUN: llvm-objdump -d %t_read_base | FileCheck %s --check-prefix=BASE
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-use-path=%t_cgdata_suffix -append-content-hash-outlined-name=true -filetype=obj %t/local-one.ll -o %t_read_suffix
+; RUN: llvm-objdump -d %t_read_suffix | FileCheck %s --check-prefix=SUFFIX
+
+;--- local-two.ll
+declare i32 @g(i32, i32, i32)
+define i32 @f1() minsize {
+ %1 = call i32 @g(i32 10, i32 1, i32 2);
+ ret i32 %1
+}
+define i32 @f2() minsize {
+ %1 = call i32 @g(i32 20, i32 1, i32 2);
+ ret i32 %1
+}
+
+;--- local-one.ll
+declare i32 @g(i32, i32, i32)
+define i32 @f3() minsize {
+ %1 = call i32 @g(i32 30, i32 1, i32 2);
+ ret i32 %1
+}
diff --git a/llvm/test/CodeGen/AArch64/cgdata-read-double-outline.ll b/llvm/test/CodeGen/AArch64/cgdata-read-double-outline.ll
new file mode 100644
index 00000000000000..49f417e9d01294
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-read-double-outline.ll
@@ -0,0 +1,57 @@
+; This test demonstrates how identical instruction sequences are handled during global outlining.
+; Currently, we do not attempt to share an outlined function for identical sequences.
+; Instead, each instruction sequence that matches against the global outlined hash tree
+; is outlined into its own unique function.
+
+; RUN: split-file %s %t
+
+; First, we generate the cgdata file from a local outline instance present in local-two.ll.
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-generate=true -filetype=obj %t/local-two.ll -o %t_write
+; RUN: llvm-cgdata merge %t_write -o %t_cgdata
+; RUN: llvm-cgdata show %t_cgdata | FileCheck %s --check-prefix=SHOW
+
+; SHOW: Outlined hash tree:
+; SHOW-NEXT: Total Node Count: 4
+; SHOW-NEXT: Terminal Node Count: 1
+; SHOW-NEXT: Depth: 3
+
+; Now, we read the cgdata for local-two-another.ll and proceed to optimistically outline
+; each instruction sequence that matches against the global outlined hash tree.
+; Since each matching sequence is considered a candidate, we expect to generate two
+; unique outlined functions. These functions, although unique, will be identical in code,
+; and thus, will be folded by the linker.
+
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-use-path=%t_cgdata -filetype=obj %t/local-two-another.ll -o %t_read
+; RUN: llvm-objdump -d %t_read | FileCheck %s
+
+; CHECK: _OUTLINED_FUNCTION_{{.*}}:
+; CHECK-NEXT: mov
+; CHECK-NEXT: mov
+; CHECK-NEXT: b
+
+; CHECK: _OUTLINED_FUNCTION_{{.*}}:
+; CHECK-NEXT: mov
+; CHECK-NEXT: mov
+; CHECK-NEXT: b
+
+;--- local-two.ll
+declare i32 @g(i32, i32, i32)
+define i32 @f1() minsize {
+ %1 = call i32 @g(i32 10, i32 1, i32 2);
+ ret i32 %1
+}
+define i32 @f2() minsize {
+ %1 = call i32 @g(i32 20, i32 1, i32 2);
+ ret i32 %1
+}
+
+;--- local-two-another.ll
+declare i32 @g(i32, i32, i32)
+define i32 @f3() minsize {
+ %1 = call i32 @g(i32 30, i32 1, i32 2);
+ ret i32 %1
+}
+define i32 @f4() minsize {
+ %1 = call i32 @g(i32 40, i32 1, i32 2);
+ ret i32 %1
+}
diff --git a/llvm/test/CodeGen/AArch64/cgdata-read-lto-outline.ll b/llvm/test/CodeGen/AArch64/cgdata-read-lto-outline.ll
new file mode 100644
index 00000000000000..e7b49f422f118a
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-read-lto-outline.ll
@@ -0,0 +1,94 @@
+; This test is similar to cgdata-read-double-outline.ll, but it is executed with LTO (Link Time Optimization).
+; It demonstrates how identical instruction sequences are handled during global outlining.
+; Currently, we do not attempt to reuse an outlined function for identical sequences.
+; Instead, each instruction sequence that appears in the global outlined hash tree
+; is outlined into its own unique function.
+
+; RUN: split-file %s %t
+
+; We first create the cgdata file from a local outline instance in local-two.ll
+; RUN: opt -module-summary %t/local-two.ll -o %t/write.bc
+; RUN: llvm-lto2 run %t/write.bc -o %t/write \
+; RUN: -r %t/write.bc,_f1,px -r %t/write.bc,_f2,px -r %t/write.bc,_g,p \
+; RUN: -codegen-data-generate=true
+; RUN: llvm-cgdata merge %t/write.1 -o %t_cgdata
+; RUN: llvm-cgdata show %t_cgdata | FileCheck %s --check-prefix=SHOW
+
+; SHOW: Outlined hash tree:
+; SHOW-NEXT: Total Node Count: 4
+; SHOW-NEXT: Terminal Node Count: 1
+; SHOW-NEXT: Depth: 3
+
+; Now, we execute either ThinLTO or LTO by reading the cgdata for local-two-another.ll.
+; With ThinLTO, similar to the no-LTO scenario shown in cgdata-read-double-outline.ll,
+; it optimistically outlines each instruction sequence that matches against
+; the global outlined hash tree. Since each matching sequence is considered a candidate,
+; we expect to generate two unique outlined functions that will be folded
+; by the linker at a later stage.
+; However, with LTO, we do not utilize the cgdata, but instead fall back to the default
+; outliner mode. This results in a single outlined function that is
+; shared across two call-sites.
+
+; Run ThinLTO
+; RUN: opt -module-summary %t/local-two-another.ll -o %t/thinlto.bc
+; RUN: llvm-lto2 run %t/thinlto.bc -o %t/thinlto \
+; RUN: -r %t/thinlto.bc,_f3,px -r %t/thinlto.bc,_f4,px -r %t/thinlto.bc,_g,p \
+; RUN: -codegen-data-use-path=%t_cgdata
+; RUN: llvm-objdump -d %t/thinlto.1 | FileCheck %s
+
+; CHECK: _OUTLINED_FUNCTION_{{.*}}:
+; CHECK-NEXT: mov
+; CHECK-NEXT: mov
+; CHECK-NEXT: b
+; CHECK: _OUTLINED_FUNCTION_{{.*}}:
+; CHECK-NEXT: mov
+; CHECK-NEXT: mov
+; CHECK-NEXT: b
+
+; Run ThinLTO while disabling the global outliner.
+; We have a single outlined case with the default outliner.
+; RUN: llvm-lto2 run %t/thinlto.bc -o %t/thinlto-disable \
+; RUN: -r %t/thinlto.bc,_f3,px -r %t/thinlto.bc,_f4,px -r %t/thinlto.bc,_g,p \
+; RUN: -codegen-data-use-path=%t_cgdata \
+; RUN: -disable-global-outlining
+; RUN: llvm-objdump -d %t/thinlto-disable.1 | FileCheck %s --check-prefix=DISABLE
+
+; DISABLE: _OUTLINED_FUNCTION_{{.*}}:
+; DISABLE-NEXT: mov
+; DISABLE-NEXT: mov
+; DISABLE-NEXT: b
+; DISABLE-NOT: _OUTLINED_FUNCTION_{{.*}}:
+
+; Run LTO, which effectively disables the global outliner.
+; RUN: opt %t/local-two-another.ll -o %t/lto.bc
+; RUN: llvm-lto2 run %t/lto.bc -o %t/lto \
+; RUN: -r %t/lto.bc,_f3,px -r %t/lto.bc,_f4,px -r %t/lto.bc,_g,p \
+; RUN: -codegen-data-use-path=%t_cgdata
+; RUN: llvm-objdump -d %t/lto.0 | FileCheck %s --check-prefix=DISABLE
+
+;--- local-two.ll
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-darwin"
+declare i32 @g(i32, i32, i32)
+define i32 @f1() minsize {
+ %1 = call i32 @g(i32 10, i32 1, i32 2);
+ ret i32 %1
+}
+define i32 @f2() minsize {
+ %1 = call i32 @g(i32 20, i32 1, i32 2);
+ ret i32 %1
+}
+
+;--- local-two-another.ll
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-darwin"
+
+declare i32 @g(i32, i32, i32)
+define i32 @f3() minsize {
+ %1 = call i32 @g(i32 30, i32 1, i32 2);
+ ret i32 %1
+}
+define i32 @f4() minsize {
+ %1 = call i32 @g(i32 40, i32 1, i32 2);
+ ret i32 %1
+}
diff --git a/llvm/test/CodeGen/AArch64/cgdata-read-priority.ll b/llvm/test/CodeGen/AArch64/cgdata-read-priority.ll
new file mode 100644
index 00000000000000..642beb46915d73
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-read-priority.ll
@@ -0,0 +1,68 @@
+; This test verifies whether we can outline a singleton instance (i.e., an instance that does not repeat)
+; using codegen data that has been read from a previous codegen run.
+; When multiple matches occur, we prioritize the candidates using the global frequency.
+
+; RUN: split-file %s %t
+
+; First, we generate the cgdata file from local outline instances present in write1.ll and write2.ll
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-generate=true -filetype=obj %t/write1.ll -o %t_write1
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-generate=true -filetype=obj %t/write2.ll -o %t_write2
+; RUN: llvm-cgdata merge %t_write1 %t_write2 -o %t_cgdata
+; RUN: llvm-cgdata show %t_cgdata | FileCheck %s --check-prefix=SHOW
+
+; SHOW: Outlined hash tree:
+; SHOW-NEXT: Total Node Count: 8
+; SHOW-NEXT: Terminal Node Count: 2
+; SHOW-NEXT: Depth: 4
+
+; Now, we read the cgdata in the machine outliner, enabling us to optimistically
+; outline a singleton instance in read.ll that matches against the cgdata.
+; There are two matches -- (1) (mov #1, mov #2, mov #3, b) and (2) (mov #2, mov #3, b).
+; Even though sequence (1) is longer than sequence (2), the latter is outlined because it occurs more frequently in the outlined hash tree.
+
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-use-path=%t_cgdata -filetype=obj %t/read.ll -o %t_read
+; RUN: llvm-objdump -d %t_read | FileCheck %s
+
+; CHECK: _OUTLINED_FUNCTION
+; CHECK-NEXT: mov
+; CHECK-NEXT: mov
+; CHECK-NEXT: b
+
+;--- write1.ll
+; The sequence (mov #2, mov #3, b) are repeated 4 times.
+declare i32 @g(i32, i32, i32)
+define i32 @f1() minsize {
+ %1 = call i32 @g(i32 10, i32 50, i32 2, i32 3);
+ ret i32 %1
+}
+define i32 @f2() minsize {
+ %1 = call i32 @g(i32 20, i32 60, i32 2, i32 3);
+ ret i32 %1
+}
+define i32 @f3() minsize {
+ %1 = call i32 @g(i32 30, i32 70, i32 2, i32 3);
+ ret i32 %1
+}
+define i32 @f4() minsize {
+ %1 = call i32 @g(i32 40, i32 80, i32 2, i32 3);
+ ret i32 %1
+}
+
+;--- write2.ll
+; The sequence (mov #1, mov #2, mov #3, b) are repeated 2 times.
+declare i32 @g(i32, i32, i32)
+define i32 @f6() minsize {
+ %1 = call i32 @g(i32 10, i32 1, i32 2, i32 3);
+ ret i32 %1
+}
+define i32 @f7() minsize {
+ %1 = call i32 @g(i32 20, i32 1, i32 2, i32 3);
+ ret i32 %1
+}
+
+;--- read.ll
+declare i32 @g(i32, i32, i32)
+define i32 @f3() minsize {
+ %1 = call i32 @g(i32 30, i32 1, i32 2, i32 3);
+ ret i32 %1
+}
diff --git a/llvm/test/CodeGen/AArch64/cgdata-read-single-outline.ll b/llvm/test/CodeGen/AArch64/cgdata-read-single-outline.ll
new file mode 100644
index 00000000000000..2c606a5a13007e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-read-single-outline.ll
@@ -0,0 +1,42 @@
+; This test verifies whether we can outline a singleton instance (i.e., an instance that does not repeat)
+; using codegen data that has been read from a previous codegen run.
+
+; RUN: split-file %s %t
+
+; First, we generate the cgdata file from a local outline instance present in local-two.ll.
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-generate=true -filetype=obj %t/local-two.ll -o %t_write
+; RUN: llvm-cgdata merge %t_write -o %t_cgdata
+; RUN: llvm-cgdata show %t_cgdata | FileCheck %s --check-prefix=SHOW
+
+; SHOW: Outlined hash tree:
+; SHOW-NEXT: Total Node Count: 4
+; SHOW-NEXT: Terminal Node Count: 1
+; SHOW-NEXT: Depth: 3
+
+; Now, we read the cgdata in the machine outliner, enabling us to optimistically
+; outline a singleton instance in local-one.ll that matches against the cgdata.
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-use-path=%t_cgdata -filetype=obj %t/local-one.ll -o %t_read
+; RUN: llvm-objdump -d %t_read | FileCheck %s
+
+; CHECK: _OUTLINED_FUNCTION
+; CHECK-NEXT: mov
+; CHECK-NEXT: mov
+; CHECK-NEXT: b
+
+;--- local-two.ll
+declare i32 @g(i32, i32, i32)
+define i32 @f1() minsize {
+ %1 = call i32 @g(i32 10, i32 1, i32 2);
+ ret i32 %1
+}
+define i32 @f2() minsize {
+ %1 = call i32 @g(i32 20, i32 1, i32 2);
+ ret i32 %1
+}
+
+;--- local-one.ll
+declare i32 @g(i32, i32, i32)
+define i32 @f3() minsize {
+ %1 = call i32 @g(i32 30, i32 1, i32 2);
+ ret i32 %1
+}
diff --git a/llvm/test/CodeGen/AArch64/cgdata-write-outline.ll b/llvm/test/CodeGen/AArch64/cgdata-write-outline.ll
new file mode 100644
index 00000000000000..0527ec1434ba09
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cgdata-write-outline.ll
@@ -0,0 +1,51 @@
+; This test verifies whether an outlined function is encoded into the __llvm_outline section
+; when the -codegen-data-generate flag is used.
+
+; Verify whether an outlined function is always created, but only encoded into the section when the flag is used.
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-generate=true -filetype=obj %s -o %t_save
+; RUN: llvm-objdump -d %t_save | FileCheck %s
+; RUN: llvm-objdump -h %t_save | FileCheck %s --check-prefix=SECTNAME
+; RUN: llc -mtriple=arm64-apple-darwin -codegen-data-generate=false -filetype=obj %s -o %t_nosave
+; RUN: llvm-objdump -d %t_nosave | FileCheck %s
+; RUN: llvm-objdump -h %t_nosave | FileCheck %s --check-prefix=NOSECTNAME
+
+; CHECK: _OUTLINED_FUNCTION
+; CHECK-NEXT: mov
+; CHECK-NEXT: mov
+; CHECK-NEXT: b
+; SECTNAME: __llvm_outline
+; NOSECTNAME-NOT: __llvm_outline
+
+; Verify the content of cgdata after it has been processed with llvm-cgdata.
+; RUN: llvm-cgdata merge %t_save -o %t_cgdata
+; RUN: llvm-cgdata dump %t_cgdata | FileCheck %s --check-prefix=TREE
+
+; TREE: :outlined_hash_tree
+; TREE: ---
+; TREE-NEXT: 0:
+; TREE-NEXT: Hash: 0x0
+; TREE-NEXT: Terminals: 0
+; TREE-NEXT: SuccessorIds: [ 1 ]
+; TREE-NEXT: 1:
+; TREE-NEXT: Hash: {{.}}
+; TREE-NEXT: Terminals: 0
+; TREE-NEXT: SuccessorIds: [ 2 ]
+; TREE-NEXT: 2:
+; TREE-NEXT: Hash: {{.}}
+; TREE-NEXT: Terminals: 0
+; TREE-NEXT: SuccessorIds: [ 3 ]
+; TREE-NEXT: 3:
+; TREE-NEXT: Hash: {{.}}
+; TREE-NEXT: Terminals: 2
+; TREE-NEXT: SuccessorIds: [ ]
+; TREE-NEXT: ...
+
+declare i32 @g(i32, i32, i32)
+define i32 @f1() minsize {
+ %1 = call i32 @g(i32 10, i32 1, i32 2);
+ ret i32 %1
+}
+define i32 @f2() minsize {
+ %1 = call i32 @g(i32 20, i32 1, i32 2);
+ ret i32 %1
+}
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index 90472f246918f3..f02425d53b3ee9 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -20,6 +20,7 @@
; CHECK-NEXT: Machine Branch Probability Analysis
; CHECK-NEXT: Default Regalloc Eviction Advisor
; CHECK-NEXT: Default Regalloc Priority Advisor
+; CHECK-NEXT: Module summary info
; CHECK-NEXT: ModulePass Manager
; CHECK-NEXT: Pre-ISel Intrinsic Lowering
; CHECK-NEXT: FunctionPass Manager
>From 0b026f309b8eacd3790f3ab58d3b114b9e70cad0 Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee at meta.com>
Date: Thu, 25 Apr 2024 22:20:48 -0700
Subject: [PATCH 8/9] [CGData] LLD for MachO
---
lld/MachO/Config.h | 1 +
lld/MachO/Driver.cpp | 39 +++++++++++++++
lld/MachO/InputSection.h | 1 +
lld/MachO/Options.td | 2 +
lld/test/MachO/cgdata-generate.s | 83 ++++++++++++++++++++++++++++++++
5 files changed, 126 insertions(+)
create mode 100644 lld/test/MachO/cgdata-generate.s
diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index 7b45f7f4c39a1b..87f910c740e6a0 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -207,6 +207,7 @@ struct Configuration {
std::vector<SectionAlign> sectionAlignments;
std::vector<SegmentProtection> segmentProtections;
bool ltoDebugPassManager = false;
+ llvm::StringRef codegenDataGeneratePath;
bool csProfileGenerate = false;
llvm::StringRef csProfilePath;
bool pgoWarnMismatch;
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 65de531db04b75..88fb7a03e10960 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -36,6 +36,7 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/BinaryFormat/MachO.h"
#include "llvm/BinaryFormat/Magic.h"
+#include "llvm/CodeGenData/CodeGenDataWriter.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/LTO/LTO.h"
#include "llvm/Object/Archive.h"
@@ -1250,6 +1251,38 @@ static void gatherInputSections() {
}
}
+static void codegenDataGenerate() {
+ TimeTraceScope timeScope("Generating codegen data");
+
+ OutlinedHashTreeRecord globalOutlineRecord;
+ for (ConcatInputSection *isec : inputSections) {
+ if (isec->getSegName() == segment_names::data &&
+ isec->getName() == section_names::outlinedHashTree) {
+ // Read outlined hash tree from each section
+ OutlinedHashTreeRecord localOutlineRecord;
+ auto *data = isec->data.data();
+ localOutlineRecord.deserialize(data);
+
+ // Merge it to the global hash tree.
+ globalOutlineRecord.merge(localOutlineRecord);
+ }
+ }
+
+ CodeGenDataWriter Writer;
+ if (!globalOutlineRecord.empty())
+ Writer.addRecord(globalOutlineRecord);
+
+ std::error_code EC;
+ auto fileName = config->codegenDataGeneratePath;
+ assert(!fileName.empty());
+ raw_fd_ostream Output(fileName, EC, sys::fs::OF_None);
+ if (EC)
+ error("fail to create raw_fd_ostream");
+
+ if (auto E = Writer.write(Output))
+ error("fail to write CGData");
+}
+
static void foldIdenticalLiterals() {
TimeTraceScope timeScope("Fold identical literals");
// We always create a cStringSection, regardless of whether dedupLiterals is
@@ -1665,6 +1698,8 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
config->ignoreAutoLinkOptions.insert(arg->getValue());
config->strictAutoLink = args.hasArg(OPT_strict_auto_link);
config->ltoDebugPassManager = args.hasArg(OPT_lto_debug_pass_manager);
+ config->codegenDataGeneratePath =
+ args.getLastArgValue(OPT_codegen_data_generate_path);
config->csProfileGenerate = args.hasArg(OPT_cs_profile_generate);
config->csProfilePath = args.getLastArgValue(OPT_cs_profile_path);
config->pgoWarnMismatch =
@@ -1959,6 +1994,10 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
}
gatherInputSections();
+
+ if (!config->codegenDataGeneratePath.empty())
+ codegenDataGenerate();
+
if (config->callGraphProfileSort)
priorityBuilder.extractCallGraphProfile();
diff --git a/lld/MachO/InputSection.h b/lld/MachO/InputSection.h
index 0f389e50425a32..bb41cc9119aab4 100644
--- a/lld/MachO/InputSection.h
+++ b/lld/MachO/InputSection.h
@@ -353,6 +353,7 @@ constexpr const char objcMethname[] = "__objc_methname";
constexpr const char objcNonLazyCatList[] = "__objc_nlcatlist";
constexpr const char objcNonLazyClassList[] = "__objc_nlclslist";
constexpr const char objcProtoList[] = "__objc_protolist";
+constexpr const char outlinedHashTree[] = "__llvm_outline";
constexpr const char pageZero[] = "__pagezero";
constexpr const char pointers[] = "__pointers";
constexpr const char rebase[] = "__rebase";
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index 11458d92b3abe0..8f217aa774aee3 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -137,6 +137,8 @@ def no_objc_category_merging : Flag<["-"], "no_objc_category_merging">,
Group<grp_lld>;
def lto_debug_pass_manager: Flag<["--"], "lto-debug-pass-manager">,
HelpText<"Debug new pass manager">, Group<grp_lld>;
+def codegen_data_generate_path : Joined<["--"], "codegen-data-generate-path=">,
+ HelpText<"Codegen data file path">, Group<grp_lld>;
def cs_profile_generate: Flag<["--"], "cs-profile-generate">,
HelpText<"Perform context sensitive PGO instrumentation">, Group<grp_lld>;
def cs_profile_path: Joined<["--"], "cs-profile-path=">,
diff --git a/lld/test/MachO/cgdata-generate.s b/lld/test/MachO/cgdata-generate.s
new file mode 100644
index 00000000000000..36846a35e1fbe6
--- /dev/null
+++ b/lld/test/MachO/cgdata-generate.s
@@ -0,0 +1,83 @@
+# REQUIRES: aarch64
+
+# RUN: rm -rf %t; split-file %s %t
+
+# RUN: llvm-mc -filetype obj -triple arm64-apple-darwin %t/merge-1.s -o %t/merge-1.o
+# RUN: llvm-mc -filetype obj -triple arm64-apple-darwin %t/merge-2.s -o %t/merge-2.o
+# RUN: llvm-mc -filetype obj -triple arm64-apple-darwin %t/main.s -o %t/main.o
+
+# This checks if the codegen data from the linker is identical to the merged codegen data
+# from each object file, which is obtained using the llvm-cgdata tool.
+# RUN: %no-arg-lld -dylib -arch arm64 -platform_version ios 14.0 15.0 -o %t/out \
+# RUN: %t/merge-1.o %t/merge-2.o %t/main.o --codegen-data-generate-path=%t/out-cgdata
+# RUN: llvm-cgdata merge %t/merge-1.o %t/merge-2.o %t/main.o -o %t/merge-cgdata
+# RUN: diff %t/out-cgdata %t/merge-cgdata
+
+# Merge order doesn't matter. `main.o` is dropped due to missing __llvm_outline.
+# RUN: llvm-cgdata merge %t/merge-2.o %t/merge-1.o -o %t/merge-cgdata-shuffle
+# RUN: diff %t/out-cgdata %t/merge-cgdata-shuffle
+
+# We can also generate the merged codegen data from the executable that is not dead-stripped.
+# RUN: llvm-objdump -h %t/out| FileCheck %s
+CHECK: __llvm_outline
+# RUN: llvm-cgdata merge %t/out -o %t/merge-cgdata-exe
+# RUN: diff %t/merge-cgdata-exe %t/merge-cgdata
+
+# Dead-strip will remove __llvm_outline sections from the final executable.
+# But the codeden data is still correctly produced from the linker.
+# RUN: %no-arg-lld -dylib -arch arm64 -platform_version ios 14.0 15.0 -o %t/out-strip \
+# RUN: %t/merge-1.o %t/merge-2.o %t/main.o -dead_strip --codegen-data-generate-path=%t/out-cgdata-strip
+# RUN: llvm-cgdata merge %t/merge-1.o %t/merge-2.o %t/main.o -o %t/merge-cgdata-strip
+# RUN: diff %t/out-cgdata-strip %t/merge-cgdata-strip
+# RUN: diff %t/out-cgdata-strip %t/merge-cgdata
+
+# Ensure no __llvm_outline section remains in the executable.
+# RUN: llvm-objdump -h %t/out-strip | FileCheck %s --check-prefix=STRIP
+STRIP-NOT: __llvm_outline
+
+#--- merge-1.s
+# The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp
+#---
+#0:
+# Hash: 0x0
+# Terminals: 0
+# SuccessorIds: [ 1 ]
+#1:
+# Hash: 0x1
+# Terminals: 0
+# SuccessorIds: [ 2 ]
+#2:
+# Hash: 0x2
+# Terminals: 4
+# SuccessorIds: [ ]
+#...
+.section __DATA,__llvm_outline
+_data:
+.byte 0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+#--- merge-2.s
+# The .data is encoded in a binary form based on the following yaml form. See serialize() in OutlinedHashTreeRecord.cpp
+#---
+#0:
+# Hash: 0x0
+# Terminals: 0
+# SuccessorIds: [ 1 ]
+#1:
+# Hash: 0x1
+# Terminals: 0
+# SuccessorIds: [ 2 ]
+#2:
+# Hash: 0x3
+# Terminals: 5
+# SuccessorIds: [ ]
+#...
+.section __DATA,__llvm_outline
+_data:
+.byte 0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+#--- main.s
+.globl _main
+
+.text
+_main:
+ ret
>From 84682d46f981ad83075bbe57f7263f510d20a84b Mon Sep 17 00:00:00 2001
From: Kyungwoo Lee <kyulee at meta.com>
Date: Fri, 26 Apr 2024 12:58:54 -0700
Subject: [PATCH 9/9] [CGData] Clang Optinos
---
clang/include/clang/Driver/Options.td | 12 ++++++
clang/lib/Driver/ToolChains/CommonArgs.cpp | 27 +++++++++++++
clang/lib/Driver/ToolChains/Darwin.cpp | 46 ++++++++++++++++++++++
clang/test/Driver/codegen-data.c | 42 ++++++++++++++++++++
4 files changed, 127 insertions(+)
create mode 100644 clang/test/Driver/codegen-data.c
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 52d161703f965e..62e50a807f4f6e 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1834,6 +1834,18 @@ def fprofile_selected_function_group :
Visibility<[ClangOption, CC1Option]>, MetaVarName<"<i>">,
HelpText<"Partition functions into N groups using -fprofile-function-groups and select only functions in group i to be instrumented. The valid range is 0 to N-1 inclusive">,
MarshallingInfoInt<CodeGenOpts<"ProfileSelectedFunctionGroup">>;
+def fcodegen_data_generate : Joined<["-"], "fcodegen-data-generate">,
+ Group<f_Group>, Visibility<[ClangOption, CC1Option]>,
+ HelpText<"Emit codegen data into object file. LLD for MachO (for now) merges them into default.cgdata">;
+def fcodegen_data_generate_EQ : Joined<["-"], "fcodegen-data-generate=">,
+ Group<f_Group>, Visibility<[ClangOption, CC1Option]>, MetaVarName<"<directory>">,
+ HelpText<"Emit codegen data into object file. LLD for MachO (for now) merges them into <directory>/default.cgdata">;
+def fcodegen_data_use : Joined<["-"], "fcodegen-data-use">,
+ Group<f_Group>, Visibility<[ClangOption, CC1Option]>,
+ HelpText<"Use codegen data read from default.cgdata to optimize the binary">;
+def fcodegen_data_use_EQ : Joined<["-"], "fcodegen-data-use=">,
+ Group<f_Group>, Visibility<[ClangOption, CC1Option]>, MetaVarName<"<directory>">,
+ HelpText<"Use codegen data read from <directory>/default.cgdata to optimize the binary">;
def fswift_async_fp_EQ : Joined<["-"], "fswift-async-fp=">,
Group<f_Group>,
Visibility<[ClangOption, CC1Option, CC1AsOption, CLOption]>,
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index b65b96db16bd79..8490955a5ae82c 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -2752,6 +2752,33 @@ void tools::addMachineOutlinerArgs(const Driver &D,
addArg(Twine("-enable-machine-outliner=never"));
}
}
+
+ auto *CodeGenDataGenArg =
+ Args.getLastArg(options::OPT_fcodegen_data_generate,
+ options::OPT_fcodegen_data_generate_EQ);
+ auto *CodeGenDataUseArg = Args.getLastArg(options::OPT_fcodegen_data_use,
+ options::OPT_fcodegen_data_use_EQ);
+
+ // We only allow one of them to be specified.
+ if (CodeGenDataGenArg && CodeGenDataUseArg)
+ D.Diag(diag::err_drv_argument_not_allowed_with)
+ << CodeGenDataGenArg->getAsString(Args)
+ << CodeGenDataUseArg->getAsString(Args);
+
+ // For codegen data gen, the output file is passed to the linker
+ // while a boolean flag is passed to the LLVM backend.
+ if (CodeGenDataGenArg)
+ addArg(Twine("-codegen-data-generate"));
+
+ // For codegen data use, the input file is passed to the LLVM backend.
+ if (CodeGenDataUseArg) {
+ SmallString<128> Path(CodeGenDataUseArg->getNumValues() == 0
+ ? ""
+ : CodeGenDataUseArg->getValue());
+ if (Path.empty() || llvm::sys::fs::is_directory(Path))
+ llvm::sys::path::append(Path, "default.cgdata");
+ addArg(Twine("-codegen-data-use-path=" + Path.str()));
+ }
}
void tools::addOpenMPDeviceRTL(const Driver &D,
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index caf6c4a444fdce..85407ea5c69dfc 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -476,6 +476,19 @@ void darwin::Linker::AddLinkArgs(Compilation &C, const ArgList &Args,
llvm::sys::path::append(Path, "default.profdata");
CmdArgs.push_back(Args.MakeArgString(Twine("--cs-profile-path=") + Path));
}
+
+ auto *CodeGenDataGenArg =
+ Args.getLastArg(options::OPT_fcodegen_data_generate,
+ options::OPT_fcodegen_data_generate_EQ);
+ if (CodeGenDataGenArg) {
+ SmallString<128> Path(CodeGenDataGenArg->getNumValues() == 0
+ ? ""
+ : CodeGenDataGenArg->getValue());
+ if (Path.empty() || llvm::sys::fs::is_directory(Path))
+ llvm::sys::path::append(Path, "default.cgdata");
+ CmdArgs.push_back(
+ Args.MakeArgString(Twine("--codegen-data-generate-path=") + Path));
+ }
}
}
@@ -633,6 +646,39 @@ void darwin::Linker::ConstructJob(Compilation &C, const JobAction &JA,
CmdArgs.push_back("-mllvm");
CmdArgs.push_back("-enable-linkonceodr-outlining");
+ // Propagate codegen data flags to the linker for the LLVM backend.
+ auto *CodeGenDataGenArg =
+ Args.getLastArg(options::OPT_fcodegen_data_generate,
+ options::OPT_fcodegen_data_generate_EQ);
+ auto *CodeGenDataUseArg = Args.getLastArg(options::OPT_fcodegen_data_use,
+ options::OPT_fcodegen_data_use_EQ);
+
+ // We only allow one of them to be specified.
+ const Driver &D = getToolChain().getDriver();
+ if (CodeGenDataGenArg && CodeGenDataUseArg)
+ D.Diag(diag::err_drv_argument_not_allowed_with)
+ << CodeGenDataGenArg->getAsString(Args)
+ << CodeGenDataUseArg->getAsString(Args);
+
+ // For codegen data gen, the output file is passed to the linker
+ // while a boolean flag is passed to the LLVM backend.
+ if (CodeGenDataGenArg) {
+ CmdArgs.push_back("-mllvm");
+ CmdArgs.push_back("-codegen-data-generate");
+ }
+
+ // For codegen data use, the input file is passed to the LLVM backend.
+ if (CodeGenDataUseArg) {
+ SmallString<128> Path(CodeGenDataUseArg->getNumValues() == 0
+ ? ""
+ : CodeGenDataUseArg->getValue());
+ if (Path.empty() || llvm::sys::fs::is_directory(Path))
+ llvm::sys::path::append(Path, "default.cgdata");
+ CmdArgs.push_back("-mllvm");
+ CmdArgs.push_back(
+ Args.MakeArgString("-codegen-data-use-path=" + Path.str()));
+ }
+
// Setup statistics file output.
SmallString<128> StatsFile =
getStatsFileName(Args, Output, Inputs[0], getToolChain().getDriver());
diff --git a/clang/test/Driver/codegen-data.c b/clang/test/Driver/codegen-data.c
new file mode 100644
index 00000000000000..a72850afc59736
--- /dev/null
+++ b/clang/test/Driver/codegen-data.c
@@ -0,0 +1,42 @@
+// Verify only one of codegen-data flag is passed.
+// RUN: not %clang -### -S --target=aarch64-linux-gnu -fcodegen-data-generate -fcodegen-data-use %s 2>&1 | FileCheck %s --check-prefix=CONFLICT
+// RUN: not %clang -### -S --target=arm64-apple-darwin -fcodegen-data-generate -fcodegen-data-use %s 2>&1 | FileCheck %s --check-prefix=CONFLICT
+// CONFLICT: error: invalid argument '-fcodegen-data-generate' not allowed with '-fcodegen-data-use'
+
+// Verify the codegen-data-generate (boolean) flag is passed to LLVM
+// RUN: %clang -### -S --target=aarch64-linux-gnu -fcodegen-data-generate %s 2>&1| FileCheck %s --check-prefix=GENERATE
+// RUN: %clang -### -S --target=arm64-apple-darwin -fcodegen-data-generate %s 2>&1| FileCheck %s --check-prefix=GENERATE
+// GENERATE: "-mllvm" "-codegen-data-generate"
+
+// Verify the codegen-data-use-path flag (with a default value) is passed to LLVM.
+// RUN: %clang -### -S --target=aarch64-linux-gnu -fcodegen-data-use %s 2>&1| FileCheck %s --check-prefix=USE
+// RUN: %clang -### -S --target=arm64-apple-darwin -fcodegen-data-use %s 2>&1| FileCheck %s --check-prefix=USE
+// RUN: mkdir -p %t.d/some/dir
+// RUN: %clang -### -S --target=aarch64-linux-gnu -fcodegen-data-use=%t.d/some/dir %s 2>&1 | FileCheck %s --check-prefix=USE-DIR
+// RUN: %clang -### -S --target=arm64-apple-darwin -fcodegen-data-use=%t.d/some/dir %s 2>&1 | FileCheck %s --check-prefix=USE-DIR
+// RUN: %clang -### -S --target=aarch64-linux-gnu -fcodegen-data-use=file %s 2>&1 | FileCheck %s --check-prefix=USE-FILE
+// RUN: %clang -### -S --target=arm64-apple-darwin -fcodegen-data-use=file %s 2>&1 | FileCheck %s --check-prefix=USE-FILE
+// USE: "-mllvm" "-codegen-data-use-path=default.cgdata"
+// USE-DIR: "-mllvm" "-codegen-data-use-path={{.*}}.d/some/dir{{/|\\\\}}default.cgdata"
+// USE-FILE: "-mllvm" "-codegen-data-use-path=file"
+
+// Verify the codegen-data-generate (boolean) flag with a LTO.
+// RUN: %clang -### -flto --target=aarch64-linux-gnu -fcodegen-data-generate %s 2>&1 | FileCheck %s --check-prefix=GENERATE-LTO
+// GENERATE-LTO: {{ld(.exe)?"}}
+// GENERATE-LTO-SAME: "-plugin-opt=-codegen-data-generate"
+// RUN: %clang -### -flto --target=arm64-apple-darwin -fcodegen-data-generate %s 2>&1 | FileCheck %s --check-prefix=GENERATE-LTO-DARWIN
+// GENERATE-LTO-DARWIN: {{ld(.exe)?"}}
+// GENERATE-LTO-DARWIN-SAME: "-mllvm" "-codegen-data-generate"
+
+// Verify the codegen-data-use-path flag with a LTO is passed to LLVM.
+// RUN: %clang -### -flto=thin --target=aarch64-linux-gnu -fcodegen-data-use %s 2>&1 | FileCheck %s --check-prefix=USE-LTO
+// USE-LTO: {{ld(.exe)?"}}
+// USE-LTO-SAME: "-plugin-opt=-codegen-data-use-path=default.cgdata"
+// RUN: %clang -### -flto=thin --target=arm64-apple-darwin -fcodegen-data-use %s 2>&1 | FileCheck %s --check-prefix=USE-LTO-DARWIN
+// USE-LTO-DARWIN: {{ld(.exe)?"}}
+// USE-LTO-DARWIN-SAME: "-mllvm" "-codegen-data-use-path=default.cgdata"
+
+// For now, LLD MachO supports for generating the codegen data at link time.
+// RUN: %clang -### -fuse-ld=lld -B%S/Inputs/lld --target=arm64-apple-darwin -fcodegen-data-generate %s 2>&1 | FileCheck %s --check-prefix=GENERATE-LLD-DARWIN
+// GENERATE-LLD-DARWIN: {{ld(.exe)?"}}
+// GENERATE-LLD-DARWIN-SAME: "--codegen-data-generate-path=default.cgdata"
More information about the llvm-commits
mailing list