[clang] [compiler-rt] [llvm] [XRay][RISCV] RISCV support for XRay (PR #117368)
Min-Yih Hsu via cfe-commits
cfe-commits at lists.llvm.org
Fri Nov 22 11:47:03 PST 2024
https://github.com/mshockwave created https://github.com/llvm/llvm-project/pull/117368
Add RISC-V support for XRay. The RV64 implementation has been tested in both QEMU and in our production environment.
Currently this requires D and C extensions, but since both RV64GC and RVA22/RVA23 are becoming mainstream, I don't think this requirement will be a big problem.
Based on the previous work by @a-poduval : https://reviews.llvm.org/D117929
>From 599370a06008092f6aa883bf11600d0b66707bc0 Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu at sifive.com>
Date: Wed, 20 Nov 2024 14:37:57 -0800
Subject: [PATCH] [XRay][RISCV] RISCV support for XRay
Add RISC-V support for XRay. The RV64 implementation has been tested in
both QEMU and in our production environment.
Currently this requires D and C extensions, but since both RV64GC and
RVA22/RVA23 are becoming mainstream, I don't think this requirement will
be a big problem.
Based on Ashwin Poduval's previous work:
https://reviews.llvm.org/D117929
Co-authored-by: Ashwin Poduval <ashwin.poduval at gmail.com>
---
clang/lib/Driver/XRayArgs.cpp | 2 +
.../cmake/Modules/AllSupportedArchDefs.cmake | 2 +-
compiler-rt/lib/xray/CMakeLists.txt | 12 +
compiler-rt/lib/xray/xray_interface.cpp | 4 +
compiler-rt/lib/xray/xray_riscv.cpp | 296 ++++++++++++++++++
.../lib/xray/xray_trampoline_riscv32.S | 83 +++++
.../lib/xray/xray_trampoline_riscv64.S | 83 +++++
.../lib/xray/xray_trampoline_riscv_common.S | 97 ++++++
compiler-rt/lib/xray/xray_tsc.h | 2 +-
llvm/lib/CodeGen/XRayInstrumentation.cpp | 7 +-
llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp | 82 +++++
llvm/lib/Target/RISCV/RISCVSubtarget.h | 3 +
llvm/lib/XRay/InstrumentationMap.cpp | 3 +-
.../RISCV/xray-attribute-instrumentation.ll | 24 ++
14 files changed, 695 insertions(+), 5 deletions(-)
create mode 100644 compiler-rt/lib/xray/xray_riscv.cpp
create mode 100644 compiler-rt/lib/xray/xray_trampoline_riscv32.S
create mode 100644 compiler-rt/lib/xray/xray_trampoline_riscv64.S
create mode 100644 compiler-rt/lib/xray/xray_trampoline_riscv_common.S
create mode 100644 llvm/test/CodeGen/RISCV/xray-attribute-instrumentation.ll
diff --git a/clang/lib/Driver/XRayArgs.cpp b/clang/lib/Driver/XRayArgs.cpp
index de5c38ebc3abbd..f8c213334a2b40 100644
--- a/clang/lib/Driver/XRayArgs.cpp
+++ b/clang/lib/Driver/XRayArgs.cpp
@@ -51,6 +51,8 @@ XRayArgs::XRayArgs(const ToolChain &TC, const ArgList &Args) {
case llvm::Triple::mips64:
case llvm::Triple::mips64el:
case llvm::Triple::systemz:
+ case llvm::Triple::riscv32:
+ case llvm::Triple::riscv64:
break;
default:
D.Diag(diag::err_drv_unsupported_opt_for_target)
diff --git a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
index b29ae179c2b4f4..5a1e8db61023b0 100644
--- a/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
+++ b/compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake
@@ -102,7 +102,7 @@ if(APPLE)
set(ALL_XRAY_SUPPORTED_ARCH ${X86_64} ${ARM64})
else()
set(ALL_XRAY_SUPPORTED_ARCH ${X86_64} ${ARM32} ${ARM64} ${MIPS32} ${MIPS64}
- powerpc64le ${HEXAGON} ${LOONGARCH64})
+ powerpc64le ${HEXAGON} ${LOONGARCH64} ${RISCV32} ${RISCV64})
endif()
set(ALL_XRAY_DSO_SUPPORTED_ARCH ${X86_64} ${ARM64})
set(ALL_SHADOWCALLSTACK_SUPPORTED_ARCH ${ARM64})
diff --git a/compiler-rt/lib/xray/CMakeLists.txt b/compiler-rt/lib/xray/CMakeLists.txt
index 7e3f1a0aa616e5..e7f01a2f4f1640 100644
--- a/compiler-rt/lib/xray/CMakeLists.txt
+++ b/compiler-rt/lib/xray/CMakeLists.txt
@@ -96,6 +96,16 @@ set(hexagon_SOURCES
xray_trampoline_hexagon.S
)
+set(riscv32_SOURCES
+ xray_riscv.cpp
+ xray_trampoline_riscv32.S
+ )
+
+set(riscv64_SOURCES
+ xray_riscv.cpp
+ xray_trampoline_riscv64.S
+ )
+
set(XRAY_SOURCE_ARCHS
arm
armhf
@@ -156,6 +166,8 @@ set(XRAY_ALL_SOURCE_FILES
${mips64_SOURCES}
${mips64el_SOURCES}
${powerpc64le_SOURCES}
+ ${riscv32_SOURCES}
+ ${riscv64_SOURCES}
${XRAY_IMPL_HEADERS}
)
list(REMOVE_DUPLICATES XRAY_ALL_SOURCE_FILES)
diff --git a/compiler-rt/lib/xray/xray_interface.cpp b/compiler-rt/lib/xray/xray_interface.cpp
index b6f0e6762f1681..e66736d9a344e1 100644
--- a/compiler-rt/lib/xray/xray_interface.cpp
+++ b/compiler-rt/lib/xray/xray_interface.cpp
@@ -57,6 +57,10 @@ static const int16_t cSledLength = 64;
static const int16_t cSledLength = 8;
#elif defined(__hexagon__)
static const int16_t cSledLength = 20;
+#elif SANITIZER_RISCV64
+static const int16_t cSledLength = 76;
+#elif defined(__riscv) && (__riscv_xlen == 32)
+static const int16_t cSledLength = 60;
#else
#error "Unsupported CPU Architecture"
#endif /* CPU architecture */
diff --git a/compiler-rt/lib/xray/xray_riscv.cpp b/compiler-rt/lib/xray/xray_riscv.cpp
new file mode 100644
index 00000000000000..89ce9305ef3dbe
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_riscv.cpp
@@ -0,0 +1,296 @@
+//===-- xray_riscv.cpp ----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// Implementation of riscv-specific routines (32- and 64-bit).
+//
+//===----------------------------------------------------------------------===//
+#include "sanitizer_common/sanitizer_common.h"
+#include "xray_defs.h"
+#include "xray_interface_internal.h"
+#include <atomic>
+
+namespace __xray {
+
+// The machine codes for some instructions used in runtime patching.
+enum PatchOpcodes : uint32_t {
+ PO_ADDI = 0x00000013, // addi rd, rs1, imm
+ PO_ADD = 0x00000033, // add rd, rs1, rs2
+ PO_SW = 0x00002023, // sw rt, base(offset)
+ PO_SD = 0x00003023, // sd rt, base(offset)
+ PO_LUI = 0x00000037, // lui rd, imm
+ PO_ORI = 0x00006013, // ori rd, rs1, imm
+ PO_OR = 0x00006033, // or rd, rs1, rs2
+ PO_SLLI = 0x00001013, // slli rd, rs, shamt
+ PO_SRLI = 0x00005013, // srli rd, rs, shamt
+ PO_JALR = 0x00000067, // jalr rs
+ PO_LW = 0x00002003, // lw rd, base(offset)
+ PO_LD = 0x00003003, // ld rd, base(offset)
+ PO_J = 0x0000006f, // jal #n_bytes
+ PO_NOP = 0x00000013, // nop - pseduo-instruction, same as addi x0, x0, 0
+};
+
+enum RegNum : uint32_t {
+ RN_R0 = 0x0,
+ RN_RA = 0x1,
+ RN_SP = 0x2,
+ RN_T0 = 0x5,
+ RN_T1 = 0x6,
+ RN_T2 = 0x7,
+ RN_A0 = 0xa,
+};
+
+static inline uint32_t encodeRTypeInstruction(uint32_t Opcode, uint32_t Rs1,
+ uint32_t Rs2, uint32_t Rd) {
+ return Rs2 << 20 | Rs1 << 15 | Rd << 7 | Opcode;
+}
+
+static inline uint32_t encodeITypeInstruction(uint32_t Opcode, uint32_t Rs1,
+ uint32_t Rd, uint32_t Imm) {
+ return Imm << 20 | Rs1 << 15 | Rd << 7 | Opcode;
+}
+
+static inline uint32_t encodeSTypeInstruction(uint32_t Opcode, uint32_t Rs1,
+ uint32_t Rs2, uint32_t Imm) {
+ uint32_t imm_msbs = (Imm & 0xfe0) << 25;
+ uint32_t imm_lsbs = (Imm & 0x01f) << 7;
+ return imm_msbs | Rs2 << 20 | Rs1 << 15 | imm_lsbs | Opcode;
+}
+
+static inline uint32_t encodeUTypeInstruction(uint32_t Opcode, uint32_t Rd,
+ uint32_t Imm) {
+ return Imm << 12 | Rd << 7 | Opcode;
+}
+
+static inline uint32_t encodeJTypeInstruction(uint32_t Opcode, uint32_t Rd,
+ uint32_t Imm) {
+ uint32_t imm_msb = (Imm & 0x80000) << 31;
+ uint32_t imm_lsbs = (Imm & 0x003ff) << 21;
+ uint32_t imm_11 = (Imm & 0x00400) << 20;
+ uint32_t imm_1912 = (Imm & 0x7f800) << 12;
+ return imm_msb | imm_lsbs | imm_11 | imm_1912 | Rd << 7 | Opcode;
+}
+
+#if SANITIZER_RISCV64
+static uint32_t hi20(uint64_t val) { return (val + 0x800) >> 12; }
+static uint32_t lo12(uint64_t val) { return val & 0xfff; }
+#elif defined(__riscv) && (__riscv_xlen == 32)
+static uint32_t hi20(uint32_t val) { return (val + 0x800) >> 12; }
+static uint32_t lo12(uint32_t val) { return val & 0xfff; }
+#endif
+
+static inline bool patchSled(const bool Enable, const uint32_t FuncId,
+ const XRaySledEntry &Sled,
+ void (*TracingHook)()) XRAY_NEVER_INSTRUMENT {
+ // When |Enable| == true,
+ // We replace the following compile-time stub (sled):
+ //
+ // xray_sled_n:
+ // J .tmpN
+ // 29 or 37 C.NOPs (58 or 74 bytes)
+ // .tmpN
+ //
+ // With one of the following runtime patches:
+ //
+ // xray_sled_n (32-bit):
+ // addi sp, sp, -16 ;create stack frame
+ // sw ra, 12(sp) ;save return address
+ // sw t2, 8(sp) ;save register t2
+ // sw t1, 4(sp) ;save register t1
+ // sw a0, 0(sp) ;save register a0
+ // lui t1, %hi(__xray_FunctionEntry/Exit)
+ // addi t1, t1, %lo(__xray_FunctionEntry/Exit)
+ // lui a0, %hi(function_id)
+ // addi a0, a0, %lo(function_id) ;pass function id
+ // jalr t1 ;call Tracing hook
+ // lw a0, 0(sp) ;restore register a0
+ // lw t1, 4(sp) ;restore register t1
+ // lw t2, 8(sp) ;restore register t2
+ // lw ra, 12(sp) ;restore return address
+ // addi sp, sp, 16 ;delete stack frame
+ //
+ // xray_sled_n (64-bit):
+ // addi sp, sp, -32 ;create stack frame
+ // sd ra, 24(sp) ;save return address
+ // sd t2, 16(sp) ;save register t2
+ // sd t1, 8(sp) ;save register t1
+ // sd a0, 0(sp) ;save register a0
+ // lui t2, %highest(__xray_FunctionEntry/Exit)
+ // addi t2, t2, %higher(__xray_FunctionEntry/Exit)
+ // slli t2, t2, 32
+ // lui t1, t1, %hi(__xray_FunctionEntry/Exit)
+ // addi t1, t1, %lo(__xray_FunctionEntry/Exit)
+ // add t1, t2, t1
+ // lui a0, %hi(function_id)
+ // addi a0, a0, %lo(function_id) ;pass function id
+ // jalr t1 ;call Tracing hook
+ // ld a0, 0(sp) ;restore register a0
+ // ld t1, 8(sp) ;restore register t1
+ // ld t2, 16(sp) ;restore register t2
+ // ld ra, 24(sp) ;restore return address
+ // addi sp, sp, 32 ;delete stack frame
+ //
+ // Replacement of the first 4-byte instruction should be the last and atomic
+ // operation, so that the user code which reaches the sled concurrently
+ // either jumps over the whole sled, or executes the whole sled when the
+ // latter is ready.
+ //
+ // When |Enable|==false, we set back the first instruction in the sled to be
+ // J 60 bytes (rv32)
+ // J 76 bytes (rv64)
+
+ uint32_t *Address = reinterpret_cast<uint32_t *>(Sled.address());
+ if (Enable) {
+ // If the ISA is RISCV 64, the Tracing Hook needs to be typecast to a 64 bit
+ // value
+#if SANITIZER_RISCV64
+ uint32_t LoTracingHookAddr = lo12(reinterpret_cast<uint64_t>(TracingHook));
+ uint32_t HiTracingHookAddr = hi20(reinterpret_cast<uint64_t>(TracingHook));
+ uint32_t HigherTracingHookAddr =
+ lo12((reinterpret_cast<uint64_t>(TracingHook) + 0x80000000) >> 32);
+ uint32_t HighestTracingHookAddr =
+ hi20((reinterpret_cast<uint64_t>(TracingHook) + 0x80000000) >> 32);
+ // We typecast the Tracing Hook to a 32 bit value for RISCV32
+#elif defined(__riscv) && (__riscv_xlen == 32)
+ uint32_t LoTracingHookAddr = lo12(reinterpret_cast<uint32_t>(TracingHook));
+ uint32_t HiTracingHookAddr = hi20((reinterpret_cast<uint32_t>(TracingHook));
+#endif
+ uint32_t LoFunctionID = lo12(FuncId);
+ uint32_t HiFunctionID = hi20(FuncId);
+ // The sled that is patched in for RISCV64 defined below. We need the entire
+ // sleds corresponding to both ISAs to be protected by defines because the
+ // first few instructions are all different, because we store doubles in
+ // case of RV64 and store words for RV32. Subsequently, we have LUI - and in
+ // case of RV64, we need extra instructions from this point on, so we see
+ // differences in addresses to which instructions are stored.
+#if SANITIZER_RISCV64
+ Address[1] = encodeSTypeInstruction(PatchOpcodes::PO_SD, RegNum::RN_SP,
+ RegNum::RN_RA, 0x18);
+ Address[2] = encodeSTypeInstruction(PatchOpcodes::PO_SD, RegNum::RN_SP,
+ RegNum::RN_T2, 0x10);
+ Address[3] = encodeSTypeInstruction(PatchOpcodes::PO_SD, RegNum::RN_SP,
+ RegNum::RN_T1, 0x8);
+ Address[4] = encodeSTypeInstruction(PatchOpcodes::PO_SD, RegNum::RN_SP,
+ RegNum::RN_A0, 0x0);
+ Address[5] = encodeUTypeInstruction(PatchOpcodes::PO_LUI, RegNum::RN_T2,
+ HighestTracingHookAddr);
+ Address[6] = encodeITypeInstruction(PatchOpcodes::PO_ADDI, RegNum::RN_T2,
+ RegNum::RN_T2, HigherTracingHookAddr);
+ Address[7] = encodeITypeInstruction(PatchOpcodes::PO_SLLI, RegNum::RN_T2,
+ RegNum::RN_T2, 0x20);
+ Address[8] = encodeUTypeInstruction(PatchOpcodes::PO_LUI, RegNum::RN_T1,
+ HiTracingHookAddr);
+ Address[9] = encodeITypeInstruction(PatchOpcodes::PO_ADDI, RegNum::RN_T1,
+ RegNum::RN_T1, LoTracingHookAddr);
+ Address[10] = encodeRTypeInstruction(PatchOpcodes::PO_ADD, RegNum::RN_T1,
+ RegNum::RN_T2, RegNum::RN_T1);
+ Address[11] = encodeUTypeInstruction(PatchOpcodes::PO_LUI, RegNum::RN_A0,
+ HiFunctionID);
+ Address[12] = encodeITypeInstruction(PatchOpcodes::PO_ADDI, RegNum::RN_A0,
+ RegNum::RN_A0, LoFunctionID);
+ Address[13] = encodeITypeInstruction(PatchOpcodes::PO_JALR, RegNum::RN_T1,
+ RegNum::RN_RA, 0x0);
+ Address[14] = encodeITypeInstruction(PatchOpcodes::PO_LD, RegNum::RN_SP,
+ RegNum::RN_A0, 0x0);
+ Address[15] = encodeITypeInstruction(PatchOpcodes::PO_LD, RegNum::RN_SP,
+ RegNum::RN_T1, 0x8);
+ Address[16] = encodeITypeInstruction(PatchOpcodes::PO_LD, RegNum::RN_SP,
+ RegNum::RN_T2, 0x10);
+ Address[17] = encodeITypeInstruction(PatchOpcodes::PO_LD, RegNum::RN_SP,
+ RegNum::RN_RA, 0x18);
+ Address[18] = encodeITypeInstruction(PatchOpcodes::PO_ADDI, RegNum::RN_SP,
+ RegNum::RN_SP, 0x20);
+ uint32_t CreateStackSpace = encodeITypeInstruction(
+ PatchOpcodes::PO_ADDI, RegNum::RN_SP, RegNum::RN_SP, 0xffe0);
+#elif defined(__riscv) && (__riscv_xlen == 32)
+ Address[1] = encodeSTypeInstruction(PatchOpcodes::PO_SW, RegNum::RN_SP,
+ RegNum::RN_RA, 0x0c);
+ Address[2] = encodeSTypeInstruction(PatchOpcodes::PO_SW, RegNum::RN_SP,
+ RegNum::RN_T2, 0x08);
+ Address[3] = encodeSTypeInstruction(PatchOpcodes::PO_SW, RegNum::RN_SP,
+ RegNum::RN_T1, 0x4);
+ Address[4] = encodeSTypeInstruction(PatchOpcodes::PO_SW, RegNum::RN_SP,
+ RegNum::RN_A0, 0x0);
+ Address[5] = encodeUTypeInstruction(PatchOpcodes::PO_LUI, RegNum::RN_T1,
+ HiTracingHookAddr);
+ Address[6] = encodeITypeInstruction(PatchOpcodes::PO_ADDI, RegNum::RN_T1,
+ RegNum::RN_T1, LoTracingHookAddr);
+ Address[7] = encodeUTypeInstruction(PatchOpcodes::PO_LUI, RegNum::RN_A0,
+ HiFunctionID);
+ Address[8] = encodeITypeInstruction(PatchOpcodes::PO_ADDI, RegNum::RN_A0,
+ RegNum::RN_A0, LoFunctionID);
+ Address[9] = encodeITypeInstruction(PatchOpcodes::PO_JALR, RegNum::RN_T1,
+ RegNum::RN_RA, 0x0);
+ Address[10] = encodeITypeInstruction(PatchOpcodes::PO_LW, RegNum::RN_SP,
+ RegNum::RN_A0, 0x0);
+ Address[11] = encodeITypeInstruction(PatchOpcodes::PO_LW, RegNum::RN_SP,
+ RegNum::RN_T1, 0x4);
+ Address[12] = encodeITypeInstruction(PatchOpcodes::PO_LW, RegNum::RN_SP,
+ RegNum::RN_T2, 0x08);
+ Address[13] = encodeITypeInstruction(PatchOpcodes::PO_LW, RegNum::RN_SP,
+ RegNum::RN_RA, 0x0c);
+ Address[14] = encodeITypeInstruction(PatchOpcodes::PO_ADDI, RegNum::RN_SP,
+ RegNum::RN_SP, 0x10);
+ uint32_t CreateStackSpace = encodeITypeInstruction(
+ PatchOpcodes::PO_ADDI, RegNum::RN_SP, RegNum::RN_SP, 0xfff0);
+#endif
+ std::atomic_store_explicit(
+ reinterpret_cast<std::atomic<uint32_t> *>(Address), CreateStackSpace,
+ std::memory_order_release);
+ } else {
+ uint32_t CreateBranch = encodeJTypeInstruction(
+ // Jump distance is different in both ISAs due to difference in size of
+ // sleds
+#if SANITIZER_RISCV64
+ PatchOpcodes::PO_J, RegNum::RN_R0,
+ 0x026); // jump encodes an offset in multiples of 2 bytes. 38*2 = 76
+#elif defined(__riscv) && (__riscv_xlen == 32)
+ PatchOpcodes::PO_J, RegNum::RN_R0,
+ 0x01e); // jump encodes an offset in multiples of 2 bytes. 30*2 = 60
+#endif
+ std::atomic_store_explicit(
+ reinterpret_cast<std::atomic<uint32_t> *>(Address), CreateBranch,
+ std::memory_order_release);
+ }
+ return true;
+}
+
+bool patchFunctionEntry(const bool Enable, const uint32_t FuncId,
+ const XRaySledEntry &Sled,
+ const XRayTrampolines &Trampolines,
+ bool LogArgs) XRAY_NEVER_INSTRUMENT {
+ // We don't support Logging argument at this moment, so we always
+ // use EntryTrampoline.
+ return patchSled(Enable, FuncId, Sled, Trampolines.EntryTrampoline);
+}
+
+bool patchFunctionExit(
+ const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+ const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+ return patchSled(Enable, FuncId, Sled, Trampolines.ExitTrampoline);
+}
+
+bool patchFunctionTailExit(
+ const bool Enable, const uint32_t FuncId, const XRaySledEntry &Sled,
+ const XRayTrampolines &Trampolines) XRAY_NEVER_INSTRUMENT {
+ return patchSled(Enable, FuncId, Sled, Trampolines.TailExitTrampoline);
+}
+
+bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
+ const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+ return false;
+}
+
+bool patchTypedEvent(const bool Enable, const uint32_t FuncId,
+ const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
+ return false;
+}
+} // namespace __xray
+
+extern "C" void __xray_ArgLoggerEntry() XRAY_NEVER_INSTRUMENT {}
diff --git a/compiler-rt/lib/xray/xray_trampoline_riscv32.S b/compiler-rt/lib/xray/xray_trampoline_riscv32.S
new file mode 100644
index 00000000000000..9916e0321d24fd
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_trampoline_riscv32.S
@@ -0,0 +1,83 @@
+//===-- xray_trampoline_riscv32.s ----------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This implements the riscv32-specific assembler for the trampolines.
+//
+//===----------------------------------------------------------------------===//
+
+.macro SAVE_ARG_REGISTERS
+ // Push argument registers to stack
+ addi sp, sp, -100
+ .cfi_def_cfa_offset 100
+ sw ra, 96(sp)
+ .cfi_offset ra, -4
+ sw a7, 92(sp)
+ sw a6, 88(sp)
+ sw a5, 84(sp)
+ sw a4, 80(sp)
+ sw a3, 76(sp)
+ sw a2, 72(sp)
+ sw a1, 68(sp)
+ sw a0, 64(sp)
+ fsd fa7, 56(sp)
+ fsd fa6, 48(sp)
+ fsd fa5, 40(sp)
+ fsd fa4, 32(sp)
+ fsd fa3, 24(sp)
+ fsd fa2, 16(sp)
+ fsd fa1, 8(sp)
+ fsd fa0, 0(sp)
+.endm
+
+.macro RESTORE_ARG_REGISTERS
+ // Restore argument registers
+ fld fa0, 0(sp)
+ fld fa1, 8(sp)
+ fld fa2, 16(sp)
+ fld fa3, 24(sp)
+ fld fa4, 32(sp)
+ fld fa5, 40(sp)
+ fld fa6, 48(sp)
+ fld fa7, 56(sp)
+ lw a0, 64(sp)
+ lw a1, 68(sp)
+ lw a2, 72(sp)
+ lw a3, 76(sp)
+ lw a4, 80(sp)
+ lw a5, 84(sp)
+ lw a6, 88(sp)
+ lw a7, 92(sp)
+ lw ra, 96(sp)
+ addi sp, sp, 100
+.endm
+
+.macro SAVE_RET_REGISTERS
+ // Push return registers to stack
+ addi sp, sp, -28
+ .cfi_def_cfa_offset 28
+ sw ra, 24(sp)
+ .cfi_offset ra, -4
+ sw a1, 20(sp)
+ sw a0, 16(sp)
+ fsd fa1, 8(sp)
+ fsd fa0, 0(sp)
+.endm
+
+.macro RESTORE_RET_REGISTERS
+ // Restore return registers
+ fld fa0, 0(sp)
+ fld fa1, 8(sp)
+ lw a0, 16(sp)
+ lw a1, 20(sp)
+ lw ra, 24(sp)
+ addi sp, sp, 28
+.endm
+
+#include "xray_trampoline_riscv_common.S"
diff --git a/compiler-rt/lib/xray/xray_trampoline_riscv64.S b/compiler-rt/lib/xray/xray_trampoline_riscv64.S
new file mode 100644
index 00000000000000..102b9881567d9a
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_trampoline_riscv64.S
@@ -0,0 +1,83 @@
+//===-- xray_trampoline_riscv64.s ----------------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This implements the riscv64-specific assembler for the trampolines.
+//
+//===----------------------------------------------------------------------===//
+
+.macro SAVE_ARG_REGISTERS
+ // Push return registers to stack
+ addi sp, sp, -136
+ .cfi_def_cfa_offset 136
+ sd ra, 128(sp)
+ .cfi_offset ra, -8
+ sd a7, 120(sp)
+ sd a6, 112(sp)
+ sd a5, 104(sp)
+ sd a4, 96(sp)
+ sd a3, 88(sp)
+ sd a2, 80(sp)
+ sd a1, 72(sp)
+ sd a0, 64(sp)
+ fsd fa7, 56(sp)
+ fsd fa6, 48(sp)
+ fsd fa5, 40(sp)
+ fsd fa4, 32(sp)
+ fsd fa3, 24(sp)
+ fsd fa2, 16(sp)
+ fsd fa1, 8(sp)
+ fsd fa0, 0(sp)
+.endm
+
+.macro SAVE_RET_REGISTERS
+ // Push return registers to stack
+ addi sp, sp, -40
+ .cfi_def_cfa_offset 40
+ sd ra, 32(sp)
+ .cfi_offset ra, -8
+ sd a1, 24(sp)
+ sd a0, 16(sp)
+ fsd fa1, 8(sp)
+ fsd fa0, 0(sp)
+.endm
+
+.macro RESTORE_RET_REGISTERS
+ // Restore return registers
+ fld fa0, 0(sp)
+ fld fa1, 8(sp)
+ ld a0, 16(sp)
+ ld a1, 24(sp)
+ ld ra, 32(sp)
+ addi sp, sp, 40
+.endm
+
+.macro RESTORE_ARG_REGISTERS
+ // Restore argument registers
+ fld fa0, 0(sp)
+ fld fa1, 8(sp)
+ fld fa2, 16(sp)
+ fld fa3, 24(sp)
+ fld fa4, 32(sp)
+ fld fa5, 40(sp)
+ fld fa6, 48(sp)
+ fld fa7, 56(sp)
+ ld a0, 64(sp)
+ ld a1, 72(sp)
+ ld a2, 80(sp)
+ ld a3, 88(sp)
+ ld a4, 96(sp)
+ ld a5, 104(sp)
+ ld a6, 112(sp)
+ ld a7, 120(sp)
+ ld ra, 128(sp)
+ addi sp, sp, 136
+.endm
+
+#include "xray_trampoline_riscv_common.S"
diff --git a/compiler-rt/lib/xray/xray_trampoline_riscv_common.S b/compiler-rt/lib/xray/xray_trampoline_riscv_common.S
new file mode 100644
index 00000000000000..7ce6fa0dcda31b
--- /dev/null
+++ b/compiler-rt/lib/xray/xray_trampoline_riscv_common.S
@@ -0,0 +1,97 @@
+//===-- xray_trampoline_riscv_common.s --------------------------*- ASM -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file is a part of XRay, a dynamic runtime instrumentation system.
+//
+// This implements the trampolines code shared between riscv32 and riscv64.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../builtins/assembly.h"
+#include "../sanitizer_common/sanitizer_asm.h"
+
+ .text
+ .p2align 2
+ .global ASM_SYMBOL(__xray_FunctionEntry)
+ ASM_TYPE_FUNCTION(__xray_FunctionEntry)
+ASM_SYMBOL(__xray_FunctionEntry):
+ CFI_STARTPROC
+ SAVE_ARG_REGISTERS
+
+ // Load the handler function pointer into a2
+ la a2, ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+ ld a2, 0(a2)
+
+ // Handler address will be null if it is not set
+ beq a2, x0, 1f
+
+ // If we reach here, we are tracing an event
+ // a0 already contains function id
+ // a1 = 0 means we are tracing an entry event
+ mv a1, x0
+ jalr a2
+
+1:
+ RESTORE_ARG_REGISTERS
+ jr ra
+ ASM_SIZE(__xray_FunctionEntry)
+ CFI_ENDPROC
+
+ .text
+ .p2align 2
+ .global ASM_SYMBOL(__xray_FunctionExit)
+ ASM_TYPE_FUNCTION(__xray_FunctionExit)
+ASM_SYMBOL(__xray_FunctionExit):
+ CFI_STARTPROC
+ SAVE_RET_REGISTERS
+
+ // Load the handler function pointer into a2
+ la a2, ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+ ld a2, 0(a2)
+
+ // Handler address will be null if it is not set
+ beq a2, x0, 1f
+
+ // If we reach here, we are tracing an event
+ // a0 already contains function id
+ // a1 = 1 means we are tracing an exit event
+ addi a1, x0, 1
+ jalr a2
+
+1:
+ RESTORE_RET_REGISTERS
+ jr ra
+ ASM_SIZE(__xray_FunctionExit)
+ CFI_ENDPROC
+
+ .text
+ .p2align 2
+ .global ASM_SYMBOL(__xray_FunctionTailExit)
+ ASM_TYPE_FUNCTION(__xray_FunctionTailExit)
+ASM_SYMBOL(__xray_FunctionTailExit):
+ CFI_STARTPROC
+ SAVE_ARG_REGISTERS
+
+ // Load the handler function pointer into a2
+ la a2, ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+ ld a2, 0(a2)
+
+ // Handler address will be null if it is not set
+ beq a2, x0, 1f
+
+ // If we reach here, we are tracing an event
+ // a0 already contains function id
+ // a1 = 2 means we are tracing a tail exit event
+ addi a1, x0, 2
+ jalr a2
+
+1:
+ RESTORE_ARG_REGISTERS
+ jr ra
+ ASM_SIZE(__xray_FunctionTailExit)
+ CFI_ENDPROC
diff --git a/compiler-rt/lib/xray/xray_tsc.h b/compiler-rt/lib/xray/xray_tsc.h
index e1cafe1bf11d2d..b62a686d6ce0f2 100644
--- a/compiler-rt/lib/xray/xray_tsc.h
+++ b/compiler-rt/lib/xray/xray_tsc.h
@@ -43,7 +43,7 @@ inline uint64_t getTSCFrequency() XRAY_NEVER_INSTRUMENT {
#elif defined(__powerpc64__)
#include "xray_powerpc64.inc"
#elif defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \
- defined(__hexagon__) || defined(__loongarch_lp64)
+ defined(__hexagon__) || defined(__loongarch_lp64) || defined(__riscv)
// Emulated TSC.
// There is no instruction like RDTSCP in user mode on ARM. ARM's CP15 does
// not have a constant frequency like TSC on x86(_64), it may go faster
diff --git a/llvm/lib/CodeGen/XRayInstrumentation.cpp b/llvm/lib/CodeGen/XRayInstrumentation.cpp
index 8f718d884cd067..1191d70f77f3e0 100644
--- a/llvm/lib/CodeGen/XRayInstrumentation.cpp
+++ b/llvm/lib/CodeGen/XRayInstrumentation.cpp
@@ -233,10 +233,13 @@ bool XRayInstrumentation::runOnMachineFunction(MachineFunction &MF) {
case Triple::ArchType::mips:
case Triple::ArchType::mipsel:
case Triple::ArchType::mips64:
- case Triple::ArchType::mips64el: {
+ case Triple::ArchType::mips64el:
+ case Triple::ArchType::riscv32:
+ case Triple::ArchType::riscv64: {
// For the architectures which don't have a single return instruction
InstrumentationOptions op;
- op.HandleTailcall = false;
+ // RISC-V support patching tail calls.
+ op.HandleTailcall = MF.getTarget().getTargetTriple().isRISCV();
op.HandleAllReturns = true;
prependRetWithPatchableExit(MF, TII, op);
break;
diff --git a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
index 0d818bc837fb70..5382751b0d8502 100644
--- a/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -113,6 +113,12 @@ class RISCVAsmPrinter : public AsmPrinter {
void emitNTLHint(const MachineInstr *MI);
+ // XRay Support
+ void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr *MI);
+ void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr *MI);
+ void LowerPATCHABLE_TAIL_CALL(const MachineInstr *MI);
+ void emitSled(const MachineInstr *MI, SledKind Kind);
+
bool lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
};
}
@@ -316,6 +322,22 @@ void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
return LowerPATCHPOINT(*OutStreamer, SM, *MI);
case TargetOpcode::STATEPOINT:
return LowerSTATEPOINT(*OutStreamer, SM, *MI);
+ case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
+ // patchable-function-entry is handled in lowerToMCInst
+ // Therefore, we break out of the switch statement if we encounter it here.
+ const Function &F = MI->getParent()->getParent()->getFunction();
+ if (F.hasFnAttribute("patchable-function-entry"))
+ break;
+
+ LowerPATCHABLE_FUNCTION_ENTER(MI);
+ return;
+ }
+ case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
+ LowerPATCHABLE_FUNCTION_EXIT(MI);
+ return;
+ case TargetOpcode::PATCHABLE_TAIL_CALL:
+ LowerPATCHABLE_TAIL_CALL(MI);
+ return;
}
MCInst OutInst;
@@ -453,11 +475,71 @@ bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
SetupMachineFunction(MF);
emitFunctionBody();
+ // Emit the XRay table
+ emitXRayTable();
+
if (EmittedOptionArch)
RTS.emitDirectiveOptionPop();
return false;
}
+void RISCVAsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr *MI) {
+ emitSled(MI, SledKind::FUNCTION_ENTER);
+}
+
+void RISCVAsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr *MI) {
+ emitSled(MI, SledKind::FUNCTION_EXIT);
+}
+
+void RISCVAsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr *MI) {
+ emitSled(MI, SledKind::TAIL_CALL);
+}
+
+void RISCVAsmPrinter::emitSled(const MachineInstr *MI, SledKind Kind) {
+ // We want to emit the jump instruction and the nops constituting the sled.
+ // The format is as follows:
+ // .Lxray_sled_N
+ // ALIGN
+ // J .tmpN
+ // 29 or 37 C.NOP instructions
+ // .tmpN
+
+ // The following variable holds the count of the number of NOPs to be patched
+ // in for XRay instrumentation during compilation.
+ // Note that RV64 and RV32 each has a sled of 76 and 60 bytes, respectively.
+ // Assuming we're using JAL to jump to .tmpN, then we only need
+ // (76 - 4)/2 = 36 NOPs for RV64 and (60 - 4)/2 = 28 for RV32. However, there
+ // is a chance that we'll use C.JAL instead, so an additional NOP is needed.
+ const uint8_t NoopsInSledCount =
+ MI->getParent()->getParent()->getSubtarget<RISCVSubtarget>().is64Bit()
+ ? 37
+ : 29;
+
+ OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo());
+ auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
+ OutStreamer->emitLabel(CurSled);
+ auto Target = OutContext.createTempSymbol();
+
+ const MCExpr *TargetExpr = MCSymbolRefExpr::create(
+ Target, MCSymbolRefExpr::VariantKind::VK_None, OutContext);
+
+ // Emit "J bytes" instruction, which jumps over the nop sled to the actual
+ // start of function.
+ EmitToStreamer(
+ *OutStreamer,
+ MCInstBuilder(RISCV::JAL).addReg(RISCV::X0).addExpr(TargetExpr));
+
+ // Emit NOP instructions
+ for (int8_t I = 0; I < NoopsInSledCount; ++I)
+ EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI)
+ .addReg(RISCV::X0)
+ .addReg(RISCV::X0)
+ .addImm(0));
+
+ OutStreamer->emitLabel(Target);
+ recordSled(CurSled, *MI, Kind, 2);
+}
+
void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) {
RISCVTargetStreamer &RTS =
static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index f2c0a3d85c998a..f8a8e03fd28d16 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -223,6 +223,9 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
return UserReservedRegister[i];
}
+ // XRay support - require D and C extensions.
+ bool isXRaySupported() const override { return hasStdExtD() && hasStdExtC(); }
+
// Vector codegen related methods.
bool hasVInstructions() const { return HasStdExtZve32x; }
bool hasVInstructionsI64() const { return HasStdExtZve64x; }
diff --git a/llvm/lib/XRay/InstrumentationMap.cpp b/llvm/lib/XRay/InstrumentationMap.cpp
index 800f0a0f47e425..0ebdcd5bac7526 100644
--- a/llvm/lib/XRay/InstrumentationMap.cpp
+++ b/llvm/lib/XRay/InstrumentationMap.cpp
@@ -63,7 +63,8 @@ loadObj(StringRef Filename, object::OwningBinary<object::ObjectFile> &ObjFile,
ObjFile.getBinary()->getArch() == Triple::loongarch64 ||
ObjFile.getBinary()->getArch() == Triple::ppc64le ||
ObjFile.getBinary()->getArch() == Triple::arm ||
- ObjFile.getBinary()->getArch() == Triple::aarch64))
+ ObjFile.getBinary()->getArch() == Triple::aarch64 ||
+ ObjFile.getBinary()->getArch() == Triple::riscv64))
return make_error<StringError>(
"File format not supported (only does ELF and Mach-O little endian "
"64-bit).",
diff --git a/llvm/test/CodeGen/RISCV/xray-attribute-instrumentation.ll b/llvm/test/CodeGen/RISCV/xray-attribute-instrumentation.ll
new file mode 100644
index 00000000000000..a8fc3bff0b0f5b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/xray-attribute-instrumentation.ll
@@ -0,0 +1,24 @@
+; RUN: llc -mtriple=riscv32-unknown-linux-gnu -mattr=+d,+c < %s | FileCheck --check-prefix=CHECK %s
+; RUN: llc -mtriple=riscv64-unknown-linux-gnu -mattr=+d,+c < %s | FileCheck --check-prefix=CHECK --check-prefix=CHECK-RISCV64 %s
+
+define i32 @foo() nounwind "function-instrument"="xray-always" {
+; CHECK: .p2align 2
+; CHECK-LABEL: .Lxray_sled_0:
+; CHECK-NEXT: j .Ltmp0
+; CHECK-COUNT-29: nop
+; CHECK-RISCV64-COUNT-8: nop
+; CHECK-LABEL: .Ltmp0:
+ ret i32 0
+; CHECK: .p2align 2
+; CHECK-LABEL: .Lxray_sled_1:
+; CHECK-NEXT: j .Ltmp1
+; CHECK-COUNT-29: nop
+; CHECK-RISCV64-COUNT-8: nop
+; CHECK-LABEL: .Ltmp1:
+; CHECK-NEXT: ret
+}
+; CHECK: .section xray_instr_map,"ao", at progbits,foo
+; CHECK-LABEL: .Lxray_sleds_start0:
+; CHECK: .Lxray_sled_0-[[TMP:.Ltmp[0-9]+]]
+; CHECK: .Lxray_sled_1-[[TMP:.Ltmp[0-9]+]]
+; CHECK-LABEL: .Lxray_sleds_end0:
More information about the cfe-commits
mailing list