[llvm] a542d54 - [X86][KCFI] Add support for memory operand unfolding
Sami Tolvanen via llvm-commits
llvm-commits at lists.llvm.org
Thu Nov 17 11:03:57 PST 2022
Author: Sami Tolvanen
Date: 2022-11-17T19:00:48Z
New Revision: a542d5422a35af2d3b5363c126f4e482fd2f98ee
URL: https://github.com/llvm/llvm-project/commit/a542d5422a35af2d3b5363c126f4e482fd2f98ee
DIFF: https://github.com/llvm/llvm-project/commit/a542d5422a35af2d3b5363c126f4e482fd2f98ee.diff
LOG: [X86][KCFI] Add support for memory operand unfolding
When the Linux kernel is compiled without -mretpoline, KCFI fails
ungracefully because it doesn't handle indirect calls with a memory
target operand. Since the KCFI check will need to load the target
address into a register for validating the type hash anyway, simply
unfold memory operands in indirect calls that need a KCFI check.
Fixes #59017
Added:
Modified:
llvm/lib/Target/X86/X86InstrCompiler.td
llvm/lib/Target/X86/X86KCFI.cpp
llvm/lib/Target/X86/X86MCInstLower.cpp
llvm/test/CodeGen/X86/kcfi.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 09e31988e5bd8..0ccf23683f0b3 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -260,7 +260,7 @@ let isPseudo = 1, SchedRW = [WriteSystem] in {
// Pseudo instructions used by KCFI.
//===----------------------------------------------------------------------===//
let
- Defs = [R10, EFLAGS] in {
+ Defs = [R10, R11, EFLAGS] in {
def KCFI_CHECK : PseudoI<
(outs), (ins GR64:$ptr, i32imm:$type), []>, Sched<[]>;
}
diff --git a/llvm/lib/Target/X86/X86KCFI.cpp b/llvm/lib/Target/X86/X86KCFI.cpp
index b4bced17048a1..4086f28804fca 100644
--- a/llvm/lib/Target/X86/X86KCFI.cpp
+++ b/llvm/lib/Target/X86/X86KCFI.cpp
@@ -63,6 +63,33 @@ bool X86KCFI::emitCheck(MachineBasicBlock &MBB,
if (MBBI->isBundled() && !std::prev(MBBI)->isBundle())
report_fatal_error("Cannot emit a KCFI check for a bundled call");
+ MachineFunction &MF = *MBB.getParent();
+ // If the call target is a memory operand, unfold it and use R11 for the
+ // call, so KCFI_CHECK won't have to recompute the address.
+ switch (MBBI->getOpcode()) {
+ case X86::CALL64m:
+ case X86::CALL64m_NT:
+ case X86::TAILJMPm64:
+ case X86::TAILJMPm64_REX: {
+ MachineBasicBlock::instr_iterator OrigCall = MBBI;
+ SmallVector<MachineInstr *, 2> NewMIs;
+ if (!TII->unfoldMemoryOperand(MF, *OrigCall, X86::R11, /*UnfoldLoad=*/true,
+ /*UnfoldStore=*/false, NewMIs))
+ report_fatal_error("Failed to unfold memory operand for a KCFI check");
+ for (auto *NewMI : NewMIs)
+ MBBI = MBB.insert(OrigCall, NewMI);
+ assert(MBBI->isCall() &&
+ "Unexpected instruction after memory operand unfolding");
+ if (OrigCall->shouldUpdateCallSiteInfo())
+ MF.moveCallSiteInfo(&*OrigCall, &*MBBI);
+ MBBI->setCFIType(MF, OrigCall->getCFIType());
+ OrigCall->eraseFromParent();
+ break;
+ }
+ default:
+ break;
+ }
+
MachineInstr *Check =
BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(X86::KCFI_CHECK))
.getInstr();
@@ -73,9 +100,6 @@ bool X86KCFI::emitCheck(MachineBasicBlock &MBB,
case X86::TAILJMPr64:
case X86::TAILJMPr64_REX:
assert(Target.isReg() && "Unexpected target operand for an indirect call");
- // KCFI_CHECK uses r10 as a temporary register.
- assert(Target.getReg() != X86::R10 &&
- "Unsupported target register for a KCFI call");
Check->addOperand(MachineOperand::CreateReg(Target.getReg(), false));
Target.setIsRenamable(false);
break;
@@ -93,7 +117,7 @@ bool X86KCFI::emitCheck(MachineBasicBlock &MBB,
}
Check->addOperand(MachineOperand::CreateImm(MBBI->getCFIType()));
- MBBI->setCFIType(*MBB.getParent(), 0);
+ MBBI->setCFIType(MF, 0);
// If not already bundled, bundle the check and the call to prevent
// further changes.
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index 157ec3631ed08..9b15bad41ff5c 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -1364,14 +1364,17 @@ void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
// and thus emitting potential call target gadgets at each indirect call
// site, load a negated constant to a register and compare that to the
// expected value at the call target.
+ const Register AddrReg = MI.getOperand(0).getReg();
const uint32_t Type = MI.getOperand(1).getImm();
- EmitAndCountInstruction(MCInstBuilder(X86::MOV32ri)
- .addReg(X86::R10D)
- .addImm(-MaskKCFIType(Type)));
+ // The check is immediately before the call. If the call target is in R10,
+ // we can clobber R11 for the check instead.
+ unsigned TempReg = AddrReg == X86::R10 ? X86::R11D : X86::R10D;
+ EmitAndCountInstruction(
+ MCInstBuilder(X86::MOV32ri).addReg(TempReg).addImm(-MaskKCFIType(Type)));
EmitAndCountInstruction(MCInstBuilder(X86::ADD32rm)
.addReg(X86::NoRegister)
- .addReg(X86::R10D)
- .addReg(MI.getOperand(0).getReg())
+ .addReg(TempReg)
+ .addReg(AddrReg)
.addImm(1)
.addReg(X86::NoRegister)
.addImm(-(PrefixNops + 4))
diff --git a/llvm/test/CodeGen/X86/kcfi.ll b/llvm/test/CodeGen/X86/kcfi.ll
index 362df82a5b20b..4b93fdddb82d7 100644
--- a/llvm/test/CodeGen/X86/kcfi.ll
+++ b/llvm/test/CodeGen/X86/kcfi.ll
@@ -37,8 +37,8 @@ define void @f1(ptr noundef %x) !kcfi_type !1 {
; MIR-LABEL: name: f1
; MIR: body:
; ISEL: CALL64r %0, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, cfi-type 12345678
-; KCFI: BUNDLE implicit-def $r10, implicit-def $r10d, implicit-def $r10w, implicit-def $r10b, implicit-def $r10bh, implicit-def $r10wh, implicit-def $eflags, implicit-def $rsp, implicit-def $esp, implicit-def $sp, implicit-def $spl, implicit-def $sph, implicit-def $hsp, implicit-def $ssp, implicit killed $rdi, implicit $rsp, implicit $ssp {
-; KCFI-NEXT: KCFI_CHECK $rdi, 12345678, implicit-def $r10, implicit-def $eflags
+; KCFI: BUNDLE{{.*}} {
+; KCFI-NEXT: KCFI_CHECK $rdi, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
; KCFI-NEXT: CALL64r killed $rdi, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp
; KCFI-NEXT: }
call void %x() [ "kcfi"(i32 12345678) ]
@@ -52,8 +52,8 @@ define void @f2(ptr noundef %x) {
; MIR-LABEL: name: f2
; MIR: body:
; ISEL: TCRETURNri64 %0, 0, csr_64, implicit $rsp, implicit $ssp, cfi-type 12345678
-; KCFI: BUNDLE implicit-def $r10, implicit-def $r10d, implicit-def $r10w, implicit-def $r10b, implicit-def $r10bh, implicit-def $r10wh, implicit-def $eflags, implicit killed $rdi, implicit $rsp, implicit $ssp {
-; KCFI-NEXT: KCFI_CHECK $rdi, 12345678, implicit-def $r10, implicit-def $eflags
+; KCFI: BUNDLE{{.*}} {
+; KCFI-NEXT: KCFI_CHECK $rdi, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
; KCFI-NEXT: TAILJMPr64 killed $rdi, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp
; KCFI-NEXT: }
tail call void %x() [ "kcfi"(i32 12345678) ]
@@ -66,9 +66,9 @@ define void @f3(ptr noundef %x) #0 {
; MIR-LABEL: name: f3
; MIR: body:
; ISEL: CALL64pcrel32 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit killed $r11, cfi-type 12345678
-; KCFI: BUNDLE implicit-def $r10, implicit-def $r10d, implicit-def $r10w, implicit-def $r10b, implicit-def $r10bh, implicit-def $r10wh, implicit-def $eflags, implicit-def $rsp, implicit-def $esp, implicit-def $sp, implicit-def $spl, implicit-def $sph, implicit-def $hsp, implicit-def $ssp, implicit killed $r11, implicit $rsp, implicit $ssp {
-; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $eflags
-; KCFI-NEXT: CALL64pcrel32 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit killed $r11
+; KCFI: BUNDLE{{.*}} {
+; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
+; KCFI-NEXT: CALL64pcrel32 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, implicit internal killed $r11
; KCFI-NEXT: }
call void %x() [ "kcfi"(i32 12345678) ]
ret void
@@ -80,9 +80,9 @@ define void @f4(ptr noundef %x) #0 {
; MIR-LABEL: name: f4
; MIR: body:
; ISEL: TCRETURNdi64 &__llvm_retpoline_r11, 0, csr_64, implicit $rsp, implicit $ssp, implicit killed $r11, cfi-type 12345678
-; KCFI: BUNDLE implicit-def $r10, implicit-def $r10d, implicit-def $r10w, implicit-def $r10b, implicit-def $r10bh, implicit-def $r10wh, implicit-def $eflags, implicit killed $r11, implicit $rsp, implicit $ssp {
-; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $eflags
-; KCFI-NEXT: TAILJMPd64 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit killed $r11
+; KCFI: BUNDLE{{.*}} {
+; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
+; KCFI-NEXT: TAILJMPd64 &__llvm_retpoline_r11, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp, implicit internal killed $r11
; KCFI-NEXT: }
tail call void %x() [ "kcfi"(i32 12345678) ]
ret void
@@ -108,6 +108,36 @@ define void @f6(ptr noundef %x) !kcfi_type !3 {
ret void
}
+ at g = external local_unnamed_addr global ptr, align 8
+
+define void @f7() {
+; MIR-LABEL: name: f7
+; MIR: body:
+; ISEL: TCRETURNmi64 killed %0, 1, $noreg, 0, $noreg, 0, csr_64, implicit $rsp, implicit $ssp, cfi-type 12345678
+; KCFI: $r11 = MOV64rm killed renamable $rax, 1, $noreg, 0, $noreg
+; KCFI-NEXT: BUNDLE{{.*}} {
+; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
+; KCFI-NEXT: TAILJMPr64 internal $r11, csr_64, implicit $rsp, implicit $ssp, implicit $rsp, implicit $ssp
+; KCFI-NEXT: }
+ %1 = load ptr, ptr @g, align 8
+ tail call void %1() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
+define void @f8() {
+; MIR-LABEL: name: f8
+; MIR: body:
+; ISEL: CALL64m killed %0, 1, $noreg, 0, $noreg, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp, cfi-type 12345678
+; KCFI: $r11 = MOV64rm killed renamable $rax, 1, $noreg, 0, $noreg
+; KCFI-NEXT: BUNDLE{{.*}} {
+; KCFI-NEXT: KCFI_CHECK $r11, 12345678, implicit-def $r10, implicit-def $r11, implicit-def $eflags
+; KCFI-NEXT: CALL64r internal $r11, csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp
+; KCFI-NEXT: }
+ %1 = load ptr, ptr @g, align 8
+ call void %1() [ "kcfi"(i32 12345678) ]
+ ret void
+}
+
attributes #0 = { "target-features"="+retpoline-indirect-branches,+retpoline-indirect-calls" }
!llvm.module.flags = !{!0}
More information about the llvm-commits
mailing list