[llvm] r313898 - [NVPTX] Implemented bar.warp.sync, barrier.sync, and vote{.sync} instructions/intrinsics/builtins.
Artem Belevich via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 21 11:44:49 PDT 2017
Author: tra
Date: Thu Sep 21 11:44:49 2017
New Revision: 313898
URL: http://llvm.org/viewvc/llvm-project?rev=313898&view=rev
Log:
[NVPTX] Implemented bar.warp.sync, barrier.sync, and vote{.sync} instructions/intrinsics/builtins.
Differential Revision: https://reviews.llvm.org/D38148
Added:
llvm/trunk/test/CodeGen/NVPTX/barrier.ll
llvm/trunk/test/CodeGen/NVPTX/vote.ll
Modified:
llvm/trunk/include/llvm/IR/IntrinsicsNVVM.td
llvm/trunk/lib/Target/NVPTX/NVPTXInstrInfo.td
llvm/trunk/lib/Target/NVPTX/NVPTXIntrinsics.td
Modified: llvm/trunk/include/llvm/IR/IntrinsicsNVVM.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/include/llvm/IR/IntrinsicsNVVM.td?rev=313898&r1=313897&r2=313898&view=diff
==============================================================================
--- llvm/trunk/include/llvm/IR/IntrinsicsNVVM.td (original)
+++ llvm/trunk/include/llvm/IR/IntrinsicsNVVM.td Thu Sep 21 11:44:49 2017
@@ -750,6 +750,17 @@ let TargetPrefix = "nvvm" in {
def int_nvvm_bar_sync :
Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>,
GCCBuiltin<"__nvvm_bar_sync">;
+ def int_nvvm_bar_warp_sync :
+ Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>,
+ GCCBuiltin<"__nvvm_bar_warp_sync">;
+
+ // barrier.sync id[, cnt]
+ def int_nvvm_barrier_sync :
+ Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>,
+ GCCBuiltin<"__nvvm_barrier_sync">;
+ def int_nvvm_barrier_sync_cnt :
+ Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent]>,
+ GCCBuiltin<"__nvvm_barrier_sync_cnt">;
// Membar
def int_nvvm_membar_cta : GCCBuiltin<"__nvvm_membar_cta">,
@@ -3780,4 +3791,55 @@ def int_nvvm_shfl_sync_idx_f32 :
Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty],
[IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.sync.idx.f32">,
GCCBuiltin<"__nvvm_shfl_sync_idx_f32">;
-}
+
+//
+// VOTE
+//
+
+// vote.all pred
+def int_nvvm_vote_all :
+ Intrinsic<[llvm_i1_ty], [llvm_i1_ty],
+ [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.all">,
+ GCCBuiltin<"__nvvm_vote_all">;
+// vote.any pred
+def int_nvvm_vote_any :
+ Intrinsic<[llvm_i1_ty], [llvm_i1_ty],
+ [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.any">,
+ GCCBuiltin<"__nvvm_vote_any">;
+// vote.uni pred
+def int_nvvm_vote_uni :
+ Intrinsic<[llvm_i1_ty], [llvm_i1_ty],
+ [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.uni">,
+ GCCBuiltin<"__nvvm_vote_uni">;
+// vote.ballot pred
+def int_nvvm_vote_ballot :
+ Intrinsic<[llvm_i32_ty], [llvm_i1_ty],
+ [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.ballot">,
+ GCCBuiltin<"__nvvm_vote_ballot">;
+
+//
+// VOTE.SYNC
+//
+
+// vote.sync.all mask, pred
+def int_nvvm_vote_all_sync :
+ Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty],
+ [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.all.sync">,
+ GCCBuiltin<"__nvvm_vote_all_sync">;
+// vote.sync.any mask, pred
+def int_nvvm_vote_any_sync :
+ Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty],
+ [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.any.sync">,
+ GCCBuiltin<"__nvvm_vote_any_sync">;
+// vote.sync.uni mask, pred
+def int_nvvm_vote_uni_sync :
+ Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty],
+ [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.uni.sync">,
+ GCCBuiltin<"__nvvm_vote_uni_sync">;
+// vote.sync.ballot mask, pred
+def int_nvvm_vote_ballot_sync :
+ Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i1_ty],
+ [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.ballot.sync">,
+ GCCBuiltin<"__nvvm_vote_ballot_sync">;
+
+} // let TargetPrefix = "nvvm"
Modified: llvm/trunk/lib/Target/NVPTX/NVPTXInstrInfo.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/NVPTX/NVPTXInstrInfo.td?rev=313898&r1=313897&r2=313898&view=diff
==============================================================================
--- llvm/trunk/lib/Target/NVPTX/NVPTXInstrInfo.td (original)
+++ llvm/trunk/lib/Target/NVPTX/NVPTXInstrInfo.td Thu Sep 21 11:44:49 2017
@@ -155,6 +155,9 @@ def noHWROT32 : Predicate<"!Subtarget->h
def true : Predicate<"true">;
def hasPTX31 : Predicate<"Subtarget->getPTXVersion() >= 31">;
+def hasPTX60 : Predicate<"Subtarget->getPTXVersion() >= 60">;
+
+def hasSM30 : Predicate<"Subtarget->getSmVersion() >= 30">;
def useFP16Math: Predicate<"Subtarget->allowFP16Math()">;
Modified: llvm/trunk/lib/Target/NVPTX/NVPTXIntrinsics.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/NVPTX/NVPTXIntrinsics.td?rev=313898&r1=313897&r2=313898&view=diff
==============================================================================
--- llvm/trunk/lib/Target/NVPTX/NVPTXIntrinsics.td (original)
+++ llvm/trunk/lib/Target/NVPTX/NVPTXIntrinsics.td Thu Sep 21 11:44:49 2017
@@ -71,6 +71,38 @@ def INT_BARRIER0_OR : NVPTXInst<(outs In
def INT_BAR_SYNC : NVPTXInst<(outs), (ins i32imm:$i), "bar.sync \t$i;",
[(int_nvvm_bar_sync imm:$i)]>;
+def INT_BAR_WARP_SYNC_I : NVPTXInst<(outs), (ins i32imm:$i), "bar.warp.sync \t$i;",
+ [(int_nvvm_bar_warp_sync imm:$i)]>,
+ Requires<[hasPTX60, hasSM30]>;
+def INT_BAR_WARP_SYNC_R : NVPTXInst<(outs), (ins Int32Regs:$i), "bar.warp.sync \t$i;",
+ [(int_nvvm_bar_warp_sync Int32Regs:$i)]>,
+ Requires<[hasPTX60, hasSM30]>;
+
+def INT_BARRIER_SYNC_I : NVPTXInst<(outs), (ins i32imm:$i), "barrier.sync \t$i;",
+ [(int_nvvm_barrier_sync imm:$i)]>,
+ Requires<[hasPTX60, hasSM30]>;
+def INT_BARRIER_SYNC_R : NVPTXInst<(outs), (ins Int32Regs:$i), "barrier.sync \t$i;",
+ [(int_nvvm_barrier_sync Int32Regs:$i)]>,
+ Requires<[hasPTX60, hasSM30]>;
+
+def INT_BARRIER_SYNC_CNT_RR : NVPTXInst<(outs), (ins Int32Regs:$id, Int32Regs:$cnt),
+ "barrier.sync \t$id, $cnt;",
+ [(int_nvvm_barrier_sync_cnt Int32Regs:$id, Int32Regs:$cnt)]>,
+ Requires<[hasPTX60, hasSM30]>;
+def INT_BARRIER_SYNC_CNT_RI : NVPTXInst<(outs), (ins Int32Regs:$id, i32imm:$cnt),
+ "barrier.sync \t$id, $cnt;",
+ [(int_nvvm_barrier_sync_cnt Int32Regs:$id, imm:$cnt)]>,
+ Requires<[hasPTX60, hasSM30]>;
+def INT_BARRIER_SYNC_CNT_IR : NVPTXInst<(outs), (ins i32imm:$id, Int32Regs:$cnt),
+ "barrier.sync \t$id, $cnt;",
+ [(int_nvvm_barrier_sync_cnt imm:$id, Int32Regs:$cnt)]>,
+ Requires<[hasPTX60, hasSM30]>;
+def INT_BARRIER_SYNC_CNT_II : NVPTXInst<(outs), (ins i32imm:$id, i32imm:$cnt),
+ "barrier.sync \t$id, $cnt;",
+ [(int_nvvm_barrier_sync_cnt imm:$id, imm:$cnt)]>,
+ Requires<[hasPTX60, hasSM30]>;
+
+
// shfl.{up,down,bfly,idx}.b32
multiclass SHFL<NVPTXRegClass regclass, string mode, Intrinsic IntOp> {
// The last two parameters to shfl can be regs or imms. ptxas is smart
@@ -184,6 +216,37 @@ defm INT_SHFL_SYNC_BFLY_F32 : SHFL_SYNC<
defm INT_SHFL_SYNC_IDX_I32 : SHFL_SYNC<Int32Regs, "idx", int_nvvm_shfl_sync_idx_i32>;
defm INT_SHFL_SYNC_IDX_F32 : SHFL_SYNC<Float32Regs, "idx", int_nvvm_shfl_sync_idx_f32>;
+
+// vote.{all,any,uni,ballot}
+multiclass VOTE<NVPTXRegClass regclass, string mode, Intrinsic IntOp> {
+ def : NVPTXInst<(outs regclass:$dest), (ins Int1Regs:$pred),
+ "vote." # mode # " \t$dest, $pred;",
+ [(set regclass:$dest, (IntOp Int1Regs:$pred))]>,
+ Requires<[hasPTX60, hasSM30]>;
+}
+
+defm VOTE_ALL : VOTE<Int1Regs, "all.pred", int_nvvm_vote_all>;
+defm VOTE_ANY : VOTE<Int1Regs, "any.pred", int_nvvm_vote_any>;
+defm VOTE_UNI : VOTE<Int1Regs, "uni.pred", int_nvvm_vote_uni>;
+defm VOTE_BALLOT : VOTE<Int32Regs, "ballot.b32", int_nvvm_vote_ballot>;
+
+// vote.sync.{all,any,uni,ballot}
+multiclass VOTE_SYNC<NVPTXRegClass regclass, string mode, Intrinsic IntOp> {
+ def i : NVPTXInst<(outs regclass:$dest), (ins i32imm:$mask, Int1Regs:$pred),
+ "vote.sync." # mode # " \t$dest, $pred, $mask;",
+ [(set regclass:$dest, (IntOp imm:$mask, Int1Regs:$pred))]>,
+ Requires<[hasPTX60, hasSM30]>;
+ def r : NVPTXInst<(outs regclass:$dest), (ins Int32Regs:$mask, Int1Regs:$pred),
+ "vote.sync." # mode #" \t$dest, $pred, $mask;",
+ [(set regclass:$dest, (IntOp Int32Regs:$mask, Int1Regs:$pred))]>,
+ Requires<[hasPTX60, hasSM30]>;
+}
+
+defm VOTE_SYNC_ALL : VOTE_SYNC<Int1Regs, "all.pred", int_nvvm_vote_all_sync>;
+defm VOTE_SYNC_ANY : VOTE_SYNC<Int1Regs, "any.pred", int_nvvm_vote_any_sync>;
+defm VOTE_SYNC_UNI : VOTE_SYNC<Int1Regs, "uni.pred", int_nvvm_vote_uni_sync>;
+defm VOTE_SYNC_BALLOT : VOTE_SYNC<Int32Regs, "ballot.b32", int_nvvm_vote_ballot_sync>;
+
} // isConvergent = 1
//-----------------------------------
Added: llvm/trunk/test/CodeGen/NVPTX/barrier.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/NVPTX/barrier.ll?rev=313898&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/NVPTX/barrier.ll (added)
+++ llvm/trunk/test/CodeGen/NVPTX/barrier.ll Thu Sep 21 11:44:49 2017
@@ -0,0 +1,32 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_30 -mattr=+ptx60 | FileCheck %s
+
+declare void @llvm.nvvm.bar.warp.sync(i32)
+declare void @llvm.nvvm.barrier.sync(i32)
+declare void @llvm.nvvm.barrier.sync.cnt(i32, i32)
+
+; CHECK-LABEL: .func{{.*}}barrier.sync
+define void @barrier.sync(i32 %id, i32 %cnt) {
+ ; CHECK: ld.param.u32 [[ID:%r[0-9]+]], [barrier.sync_param_0];
+ ; CHECK: ld.param.u32 [[CNT:%r[0-9]+]], [barrier.sync_param_1];
+
+ ; CHECK: barrier.sync [[ID]], [[CNT]];
+ call void @llvm.nvvm.barrier.sync.cnt(i32 %id, i32 %cnt)
+ ; CHECK: barrier.sync [[ID]], 2;
+ call void @llvm.nvvm.barrier.sync.cnt(i32 %id, i32 2)
+ ; CHECK: barrier.sync 3, [[CNT]];
+ call void @llvm.nvvm.barrier.sync.cnt(i32 3, i32 %cnt)
+ ; CHECK: barrier.sync 4, 5;
+ call void @llvm.nvvm.barrier.sync.cnt(i32 4, i32 5)
+
+ ; CHECK: barrier.sync [[ID]];
+ call void @llvm.nvvm.barrier.sync(i32 %id)
+ ; CHECK: barrier.sync 1;
+ call void @llvm.nvvm.barrier.sync(i32 1)
+
+ ; CHECK: bar.warp.sync [[ID]];
+ call void @llvm.nvvm.bar.warp.sync(i32 %id)
+ ; CHECK: bar.warp.sync 6;
+ call void @llvm.nvvm.bar.warp.sync(i32 6)
+ ret void;
+}
+
Added: llvm/trunk/test/CodeGen/NVPTX/vote.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/NVPTX/vote.ll?rev=313898&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/NVPTX/vote.ll (added)
+++ llvm/trunk/test/CodeGen/NVPTX/vote.ll Thu Sep 21 11:44:49 2017
@@ -0,0 +1,65 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_30 -mattr=+ptx60 | FileCheck %s
+
+declare i1 @llvm.nvvm.vote.all(i1)
+; CHECK-LABEL: .func{{.*}}vote.all
+define i1 @vote.all(i1 %pred) {
+ ; CHECK: vote.all.pred
+ %val = call i1 @llvm.nvvm.vote.all(i1 %pred)
+ ret i1 %val
+}
+
+declare i1 @llvm.nvvm.vote.any(i1)
+; CHECK-LABEL: .func{{.*}}vote.any
+define i1 @vote.any(i1 %pred) {
+ ; CHECK: vote.any.pred
+ %val = call i1 @llvm.nvvm.vote.any(i1 %pred)
+ ret i1 %val
+}
+
+declare i1 @llvm.nvvm.vote.uni(i1)
+; CHECK-LABEL: .func{{.*}}vote.uni
+define i1 @vote.uni(i1 %pred) {
+ ; CHECK: vote.uni.pred
+ %val = call i1 @llvm.nvvm.vote.uni(i1 %pred)
+ ret i1 %val
+}
+
+declare i32 @llvm.nvvm.vote.ballot(i1)
+; CHECK-LABEL: .func{{.*}}vote.ballot
+define i32 @vote.ballot(i1 %pred) {
+ ; CHECK: vote.ballot.b32
+ %val = call i32 @llvm.nvvm.vote.ballot(i1 %pred)
+ ret i32 %val
+}
+
+declare i1 @llvm.nvvm.vote.all.sync(i32, i1)
+; CHECK-LABEL: .func{{.*}}vote.sync.all
+define i1 @vote.sync.all(i32 %mask, i1 %pred) {
+ ; CHECK: vote.sync.all.pred
+ %val = call i1 @llvm.nvvm.vote.all.sync(i32 %mask, i1 %pred)
+ ret i1 %val
+}
+
+declare i1 @llvm.nvvm.vote.any.sync(i32, i1)
+; CHECK-LABEL: .func{{.*}}vote.sync.any
+define i1 @vote.sync.any(i32 %mask, i1 %pred) {
+ ; CHECK: vote.sync.any.pred
+ %val = call i1 @llvm.nvvm.vote.any.sync(i32 %mask, i1 %pred)
+ ret i1 %val
+}
+
+declare i1 @llvm.nvvm.vote.uni.sync(i32, i1)
+; CHECK-LABEL: .func{{.*}}vote.sync.uni
+define i1 @vote.sync.uni(i32 %mask, i1 %pred) {
+ ; CHECK: vote.sync.uni.pred
+ %val = call i1 @llvm.nvvm.vote.uni.sync(i32 %mask, i1 %pred)
+ ret i1 %val
+}
+
+declare i32 @llvm.nvvm.vote.ballot.sync(i32, i1)
+; CHECK-LABEL: .func{{.*}}vote.sync.ballot
+define i32 @vote.sync.ballot(i32 %mask, i1 %pred) {
+ ; CHECK: vote.sync.ballot.b32
+ %val = call i32 @llvm.nvvm.vote.ballot.sync(i32 %mask, i1 %pred)
+ ret i32 %val
+}
More information about the llvm-commits
mailing list