[llvm] [llvm][NVPTX] Don't reorder MIs that construct a PTX function call (PR #116522)
Youngsuk Kim via llvm-commits
llvm-commits at lists.llvm.org
Sat Nov 16 19:00:15 PST 2024
https://github.com/JOE1994 updated https://github.com/llvm/llvm-project/pull/116522
>From 1d32026891d804a3deab80653a4d4e5bfea5ccdd Mon Sep 17 00:00:00 2001
From: Youngsuk Kim <youngsuk.kim at hpe.com>
Date: Sat, 16 Nov 2024 20:25:58 -0600
Subject: [PATCH 1/2] [llvm][NVPTX] Don't reorder MIs that construct a PTX
function call
With "-enable-misched", MachineScheduler can reorder MIs that must stick together
(in initially set order) to generate legal PTX code for a function call.
When generating PTX code for the attached test (using LLVM before this revision),
the following invalid PTX code is generated:
```
{ // callseq 0, 0
.param .b64 param0;
st.param.f64 [param0], %fd1;
.param .b64 retval0;
call.uni (retval0),
cvt.u32.u64 %r20, %rd18;
mad.lo.s32 %r21, %r7, %r20, 1;
cvt.rn.f64.s32 %fd4, %r21;
_FOO,
(
param0
);
ld.param.f64 %fd2, [retval0];
add.s32 %r22, %r18, 1;
cvt.rn.f64.s32 %fd5, %r22;
} // callseq 0
```
---
llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp | 20 ++++
llvm/lib/Target/NVPTX/NVPTXInstrInfo.h | 3 +
llvm/test/CodeGen/NVPTX/misched_func_call.ll | 108 +++++++++++++++++++
3 files changed, 131 insertions(+)
create mode 100644 llvm/test/CodeGen/NVPTX/misched_func_call.ll
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
index bec40874c89488..9c1ed0d5f5abd9 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
@@ -202,3 +202,23 @@ unsigned NVPTXInstrInfo::insertBranch(MachineBasicBlock &MBB,
BuildMI(&MBB, DL, get(NVPTX::GOTO)).addMBB(FBB);
return 2;
}
+
+bool NVPTXInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
+ const MachineBasicBlock *MBB,
+ const MachineFunction &MF) const {
+ // Prevent the scheduler from reordering & splitting up MachineInstrs
+ // which must stick together (in initially set order) to
+ // comprise a valid PTX function call sequence.
+ switch (MI.getOpcode()) {
+ case NVPTX::CallUniPrintCallRetInst1:
+ case NVPTX::CallArgBeginInst:
+ case NVPTX::CallArgI32imm:
+ case NVPTX::CallArgParam:
+ case NVPTX::LastCallArgI32imm:
+ case NVPTX::LastCallArgParam:
+ case NVPTX::CallArgEndInst1:
+ return true;
+ }
+
+ return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
+}
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h
index f674a00bc351bf..a1d9f017120188 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.h
@@ -67,6 +67,9 @@ class NVPTXInstrInfo : public NVPTXGenInstrInfo {
MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
const DebugLoc &DL,
int *BytesAdded = nullptr) const override;
+ bool isSchedulingBoundary(const MachineInstr &MI,
+ const MachineBasicBlock *MBB,
+ const MachineFunction &MF) const override;
};
} // namespace llvm
diff --git a/llvm/test/CodeGen/NVPTX/misched_func_call.ll b/llvm/test/CodeGen/NVPTX/misched_func_call.ll
new file mode 100644
index 00000000000000..c54674a9c791f2
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/misched_func_call.ll
@@ -0,0 +1,108 @@
+; RUN: llc -O3 -march=nvptx64 -enable-misched %s -o - | FileCheck %s
+
+; ModuleID = 'The Accel Module'
+source_filename = "The Accel Module"
+target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
+target triple = "nvptx64-nvidia-cuda"
+
+; Function Attrs: noinline
+define ptx_kernel void @"my_kernel"(i32 %"arg_0", i64 %"arg_1", i64 %"arg_2", i64 %"arg_3") {
+"Entry_BB":
+%r = tail call i32 @llvm.nvvm.read.ptx.sreg.ntid.x()
+%r6 = tail call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x()
+%r7 = mul i32 %r, %r6
+%r9 = tail call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x()
+%r10 = tail call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+%r11 = mul i32 %r, %r9
+%r12 = add i32 %r10, %r11
+%"arg_3.tr" = trunc i64 %"arg_3" to i32
+%r16 = shl i32 %"arg_3.tr", 1
+%r19.not = icmp slt i32 %r12, %r16
+br i1 %r19.not, label %"BB1490", label %"EXIT_BB"
+
+"BB1490": ; preds = %"Entry_BB"
+%r23 = sext i32 %"arg_0" to i64
+%r24 = shl nsw i64 %r23, 3
+br label %"BB1692"
+
+"BB1692": ; preds = %"BB18", %"BB1490"
+%"$$i_l40_0_t23.0" = phi i32 [ %r12, %"BB1490" ], [ %r80, %"BB18" ]
+%r28 = sext i32 %"$$i_l40_0_t23.0" to i64
+%0 = or i64 %r28, %"arg_3"
+%1 = and i64 %0, -4294967296
+%2 = icmp eq i64 %1, 0
+br i1 %2, label %3, label %8
+
+3: ; preds = %"BB1692"
+%4 = trunc i64 %"arg_3" to i32
+%5 = trunc i64 %r28 to i32
+%6 = udiv i32 %5, %4
+%7 = zext i32 %6 to i64
+br label %"BB18"
+
+8: ; preds = %"BB1692"
+%9 = sdiv i64 %r28, %"arg_3"
+br label %"BB18"
+
+"BB18": ; preds = %8, %3
+%10 = phi i64 [ %7, %3 ], [ %9, %8 ]
+%r31 = trunc i64 %10 to i32
+%.neg = mul i64 %10, -4294967296
+%r35 = ashr exact i64 %.neg, 32
+%r38 = mul i64 %"arg_3", %r35
+%r39 = add i64 %r28, %r38
+%r42 = mul i64 %r24, %r39
+%r44 = mul i32 %r31, 10
+%r47 = inttoptr i64 %"arg_1" to ptr addrspace(1)
+%gep2 = getelementptr i8, ptr addrspace(1) %r47, i64 %r42
+%11 = sext i32 %r44 to i64
+%r53 = getelementptr double, ptr addrspace(1) %gep2, i64 %11
+%r54 = load double, ptr addrspace(1) %r53, align 8
+; CHECK: call.uni (retval0),
+; CHECK-NEXT: _FOO,
+; CHECK-NEXT: (
+; CHECK-NEXT: param0
+; CHECK-NEXT: );
+%r55 = tail call double @_FOO(double %r54)
+%12 = trunc i64 %r39 to i32
+%r59 = mul i32 %"arg_0", %12
+%r60 = add i32 %r59, 1
+%r61 = sitofp i32 %r60 to double
+%r65 = add i32 %r31, 1
+%r66 = sitofp i32 %r65 to double
+%r67 = tail call double @llvm.fma.f64(double %r55, double %r55, double %r66)
+%r68 = fadd double %r67, %r61
+%r71 = inttoptr i64 %"arg_2" to ptr addrspace(1)
+%gep88 = getelementptr i8, ptr addrspace(1) %r71, i64 %r42
+%r77 = getelementptr double, ptr addrspace(1) %gep88, i64 %11
+store double %r68, ptr addrspace(1) %r77, align 8
+%r80 = add i32 %r7, %"$$i_l40_0_t23.0"
+%r85 = icmp slt i32 %r80, %r16
+br i1 %r85, label %"BB1692", label %"EXIT_BB"
+
+"EXIT_BB": ; preds = %"BB18", %"Entry_BB"
+ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.nvvm.read.ptx.sreg.ntid.x() #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.nvvm.read.ptx.sreg.nctaid.x() #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() #1
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef i32 @llvm.nvvm.read.ptx.sreg.tid.x() #1
+
+declare double @_FOO(double)
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare double @llvm.fma.f64(double, double, double) #1
+
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!llvm.module.flags = !{ !2}
+
+!2 = !{i32 4, !"nvvm-reflect-ftz", i32 0}
>From e074ec7a23671755ee1b16bcd49335dc08838be2 Mon Sep 17 00:00:00 2001
From: Youngsuk Kim <youngsuk.kim at hpe.com>
Date: Sat, 16 Nov 2024 21:00:02 -0600
Subject: [PATCH 2/2] Apply clang-format
---
llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
index 9c1ed0d5f5abd9..d261f98b080105 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.cpp
@@ -210,14 +210,14 @@ bool NVPTXInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
// which must stick together (in initially set order) to
// comprise a valid PTX function call sequence.
switch (MI.getOpcode()) {
- case NVPTX::CallUniPrintCallRetInst1:
- case NVPTX::CallArgBeginInst:
- case NVPTX::CallArgI32imm:
- case NVPTX::CallArgParam:
- case NVPTX::LastCallArgI32imm:
- case NVPTX::LastCallArgParam:
- case NVPTX::CallArgEndInst1:
- return true;
+ case NVPTX::CallUniPrintCallRetInst1:
+ case NVPTX::CallArgBeginInst:
+ case NVPTX::CallArgI32imm:
+ case NVPTX::CallArgParam:
+ case NVPTX::LastCallArgI32imm:
+ case NVPTX::LastCallArgParam:
+ case NVPTX::CallArgEndInst1:
+ return true;
}
return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF);
More information about the llvm-commits
mailing list