[clang] [llvm] [AMDGPU] Update permlane_bcast/down/up/xor intrinsic to support more types (PR #197141)
Mariusz Sikora via llvm-commits
llvm-commits at lists.llvm.org
Tue May 12 03:26:58 PDT 2026
https://github.com/mariusz-sikora-at-amd created https://github.com/llvm/llvm-project/pull/197141
None
>From 2ec483d21d5a639ec3db2e8d08302df10ab91107 Mon Sep 17 00:00:00 2001
From: Acim Maravic <Acim.Maravic at amd.com>
Date: Tue, 12 May 2026 06:01:45 -0400
Subject: [PATCH] [AMDGPU] Update permlane_bcast/down/up/xor intrinsic to
support more types
---
clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp | 12 +
.../CodeGenOpenCL/builtins-amdgcn-gfx1250.cl | 18 +-
llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 24 +-
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 18 +-
.../AMDGPU/AMDGPURegBankLegalizeRules.cpp | 5 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 33 +-
llvm/lib/Target/AMDGPU/VOP3Instructions.td | 14 +-
.../UniformityAnalysis/AMDGPU/intrinsics.ll | 8 +-
.../AMDGPU/llvm.amdgcn.permlane.gfx1250.ll | 2968 ++++++++++++++++-
9 files changed, 2998 insertions(+), 102 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
index cfad312d7535a..751cd9847bd31 100644
--- a/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/AMDGPU.cpp
@@ -2205,6 +2205,18 @@ Value *CodeGenFunction::EmitAMDGPUBuiltinExpr(unsigned BuiltinID,
case Builtin::BI__builtin_scalbn:
return emitBinaryExpMaybeConstrainedFPBuiltin(
*this, E, Intrinsic::ldexp, Intrinsic::experimental_constrained_ldexp);
+ case AMDGPU::BI__builtin_amdgcn_permlane_bcast:
+ return emitBuiltinWithOneOverloadedType<3>(
+ *this, E, Intrinsic::amdgcn_permlane_bcast);
+ case AMDGPU::BI__builtin_amdgcn_permlane_up:
+ return emitBuiltinWithOneOverloadedType<3>(*this, E,
+ Intrinsic::amdgcn_permlane_up);
+ case AMDGPU::BI__builtin_amdgcn_permlane_down:
+ return emitBuiltinWithOneOverloadedType<3>(*this, E,
+ Intrinsic::amdgcn_permlane_down);
+ case AMDGPU::BI__builtin_amdgcn_permlane_xor:
+ return emitBuiltinWithOneOverloadedType<3>(*this, E,
+ Intrinsic::amdgcn_permlane_xor);
default:
return nullptr;
}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
index 0b4cdd0c2c28f..f0531bef642b0 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx1250.cl
@@ -1296,7 +1296,7 @@ void test_permlane16_swap(global uint2* out, uint old, uint src) {
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC0_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[SRC1_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[SRC2_ADDR_ASCAST]], align 4
-// CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.permlane.bcast(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]])
+// CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.permlane.bcast.i32(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]])
// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
// CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(1) [[TMP4]], align 4
// CHECK-NEXT: ret void
@@ -1322,7 +1322,7 @@ void test_permlane_bcast(global uint* out, uint src0, uint src1, uint src2) {
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC0_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[SRC1_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[SRC2_ADDR_ASCAST]], align 4
-// CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.permlane.down(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]])
+// CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.permlane.down.i32(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]])
// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
// CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(1) [[TMP4]], align 4
// CHECK-NEXT: ret void
@@ -1348,7 +1348,7 @@ void test_permlane_down(global uint* out, uint src0, uint src1, uint src2) {
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC0_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[SRC1_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[SRC2_ADDR_ASCAST]], align 4
-// CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.permlane.up(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]])
+// CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.permlane.up.i32(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]])
// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
// CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(1) [[TMP4]], align 4
// CHECK-NEXT: ret void
@@ -1374,7 +1374,7 @@ void test_permlane_up(global uint* out, uint src0, uint src1, uint src2) {
// CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC0_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[SRC1_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[SRC2_ADDR_ASCAST]], align 4
-// CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.permlane.xor(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]])
+// CHECK-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.permlane.xor.i32(i32 [[TMP0]], i32 [[TMP1]], i32 [[TMP2]])
// CHECK-NEXT: [[TMP4:%.*]] = load ptr addrspace(1), ptr [[OUT_ADDR_ASCAST]], align 8
// CHECK-NEXT: store i32 [[TMP3]], ptr addrspace(1) [[TMP4]], align 4
// CHECK-NEXT: ret void
@@ -1514,7 +1514,7 @@ void test_s_wakeup_barrier(void *bar)
// CHECK-NEXT: store float [[X:%.*]], ptr [[X_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load ptr addrspace(1), ptr [[ADDR_ADDR_ASCAST]], align 8
// CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[X_ADDR_ASCAST]], align 4
-// CHECK-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP0]], float [[TMP1]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4:![0-9]+]], !amdgpu.ignore.denormal.mode [[META4]]
+// CHECK-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP0]], float [[TMP1]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META3:![0-9]+]], !amdgpu.ignore.denormal.mode [[META3]]
// CHECK-NEXT: ret float [[TMP2]]
//
float test_global_add_f32(global float *addr, float x) {
@@ -1531,7 +1531,7 @@ float test_global_add_f32(global float *addr, float x) {
// CHECK-NEXT: store <2 x half> [[X:%.*]], ptr [[X_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load ptr addrspace(1), ptr [[ADDR_ADDR_ASCAST]], align 8
// CHECK-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[X_ADDR_ASCAST]], align 4
-// CHECK-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP0]], <2 x half> [[TMP1]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4]]
+// CHECK-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP0]], <2 x half> [[TMP1]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META3]]
// CHECK-NEXT: ret <2 x half> [[TMP2]]
//
half2 test_global_add_half2(global half2 *addr, half2 x) {
@@ -1548,7 +1548,7 @@ half2 test_global_add_half2(global half2 *addr, half2 x) {
// CHECK-NEXT: store <2 x half> [[X:%.*]], ptr [[X_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ADDR_ADDR_ASCAST]], align 8
// CHECK-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr [[X_ADDR_ASCAST]], align 4
-// CHECK-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr [[TMP0]], <2 x half> [[TMP1]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4]]
+// CHECK-NEXT: [[TMP2:%.*]] = atomicrmw fadd ptr [[TMP0]], <2 x half> [[TMP1]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META3]]
// CHECK-NEXT: ret <2 x half> [[TMP2]]
//
half2 test_flat_add_2f16(generic half2 *addr, half2 x) {
@@ -1566,7 +1566,7 @@ half2 test_flat_add_2f16(generic half2 *addr, half2 x) {
// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ADDR_ADDR_ASCAST]], align 8
// CHECK-NEXT: [[TMP1:%.*]] = load <2 x i16>, ptr [[X_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i16> [[TMP1]] to <2 x bfloat>
-// CHECK-NEXT: [[TMP3:%.*]] = atomicrmw fadd ptr [[TMP0]], <2 x bfloat> [[TMP2]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4]]
+// CHECK-NEXT: [[TMP3:%.*]] = atomicrmw fadd ptr [[TMP0]], <2 x bfloat> [[TMP2]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META3]]
// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x bfloat> [[TMP3]] to <2 x i16>
// CHECK-NEXT: ret <2 x i16> [[TMP4]]
//
@@ -1585,7 +1585,7 @@ short2 test_flat_add_2bf16(generic short2 *addr, short2 x) {
// CHECK-NEXT: [[TMP0:%.*]] = load ptr addrspace(1), ptr [[ADDR_ADDR_ASCAST]], align 8
// CHECK-NEXT: [[TMP1:%.*]] = load <2 x i16>, ptr [[X_ADDR_ASCAST]], align 4
// CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x i16> [[TMP1]] to <2 x bfloat>
-// CHECK-NEXT: [[TMP3:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP0]], <2 x bfloat> [[TMP2]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META4]]
+// CHECK-NEXT: [[TMP3:%.*]] = atomicrmw fadd ptr addrspace(1) [[TMP0]], <2 x bfloat> [[TMP2]] syncscope("agent") monotonic, align 4, !amdgpu.no.fine.grained.memory [[META3]]
// CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x bfloat> [[TMP3]] to <2 x i16>
// CHECK-NEXT: ret <2 x i16> [[TMP4]]
//
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 8631985de9a0a..63920e91ffcaf 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -3763,27 +3763,27 @@ def int_amdgcn_sat_pk4_u4_u8 : ClangBuiltin<"__builtin_amdgcn_sat_pk4_u4_u8">,
PureIntrinsic<[llvm_i16_ty], [llvm_i32_ty]>;
// llvm.amdgcn.permlane.bcast <src0> <src1> <src2>
-def int_amdgcn_permlane_bcast : ClangBuiltin<"__builtin_amdgcn_permlane_bcast">,
- Intrinsic<[llvm_i32_ty],
- [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+def int_amdgcn_permlane_bcast :
+ Intrinsic<[llvm_any_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty],
[IntrNoMem, IntrConvergent, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
// llvm.amdgcn.permlane.up <src0> <src1> <src2>
-def int_amdgcn_permlane_up : ClangBuiltin<"__builtin_amdgcn_permlane_up">,
- Intrinsic<[llvm_i32_ty],
- [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+def int_amdgcn_permlane_up :
+ Intrinsic<[llvm_any_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty],
[IntrNoMem, IntrConvergent, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
// llvm.amdgcn.permlane.down <src0> <src1> <src2>
-def int_amdgcn_permlane_down : ClangBuiltin<"__builtin_amdgcn_permlane_down">,
- Intrinsic<[llvm_i32_ty],
- [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+def int_amdgcn_permlane_down :
+ Intrinsic<[llvm_any_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty],
[IntrNoMem, IntrConvergent, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
// llvm.amdgcn.permlane.xor <src0> <src1> <src2>
-def int_amdgcn_permlane_xor : ClangBuiltin<"__builtin_amdgcn_permlane_xor">,
- Intrinsic<[llvm_i32_ty],
- [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+def int_amdgcn_permlane_xor :
+ Intrinsic<[llvm_any_ty],
+ [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty],
[IntrNoMem, IntrConvergent, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
// llvm.amdgcn.permlane.idx.gen <src0> <src1>
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 43db1ead84c80..2c21fd1209a02 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -6126,6 +6126,10 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper,
IID == Intrinsic::amdgcn_permlanex16;
bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
IID == Intrinsic::amdgcn_set_inactive_chain_arg;
+ bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
+ IID == Intrinsic::amdgcn_permlane_up ||
+ IID == Intrinsic::amdgcn_permlane_down ||
+ IID == Intrinsic::amdgcn_permlane_xor;
auto createLaneOp = [&IID, &B, &MI](Register Src0, Register Src1,
Register Src2, LLT VT) -> Register {
@@ -6139,6 +6143,10 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper,
case Intrinsic::amdgcn_set_inactive_chain_arg:
return LaneOp.addUse(Src1).getReg(0);
case Intrinsic::amdgcn_writelane:
+ case Intrinsic::amdgcn_permlane_bcast:
+ case Intrinsic::amdgcn_permlane_up:
+ case Intrinsic::amdgcn_permlane_down:
+ case Intrinsic::amdgcn_permlane_xor:
return LaneOp.addUse(Src1).addUse(Src2).getReg(0);
case Intrinsic::amdgcn_permlane16:
case Intrinsic::amdgcn_permlanex16: {
@@ -6170,9 +6178,11 @@ bool AMDGPULegalizerInfo::legalizeLaneOp(LegalizerHelper &Helper,
Register Src0 = MI.getOperand(2).getReg();
Register Src1, Src2;
if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
- IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
+ IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
+ IsPermlaneShuffle) {
Src1 = MI.getOperand(3).getReg();
- if (IID == Intrinsic::amdgcn_writelane || IsPermLane16) {
+ if (IID == Intrinsic::amdgcn_writelane || IsPermLane16 ||
+ IsPermlaneShuffle) {
Src2 = MI.getOperand(4).getReg();
}
}
@@ -8447,6 +8457,10 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
case Intrinsic::amdgcn_set_inactive_chain_arg:
case Intrinsic::amdgcn_mov_dpp8:
case Intrinsic::amdgcn_update_dpp:
+ case Intrinsic::amdgcn_permlane_bcast:
+ case Intrinsic::amdgcn_permlane_up:
+ case Intrinsic::amdgcn_permlane_down:
+ case Intrinsic::amdgcn_permlane_xor:
return legalizeLaneOp(Helper, MI, IntrID);
case Intrinsic::amdgcn_s_buffer_prefetch_data:
return legalizeSBufferPrefetch(Helper, MI);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 061b8dc070ead..c1b0e72a8c42d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -1691,7 +1691,10 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
Standard)
.Div(S32,
{{Vgpr32},
- {IntrId, Vgpr32, SgprB32_ReadFirstLane, SgprB32_ReadFirstLane}});
+ {IntrId, Vgpr32, SgprB32_ReadFirstLane, SgprB32_ReadFirstLane}})
+ .Div(V2S16,
+ {{VgprV2S16},
+ {IntrId, VgprV2S16, SgprB32_ReadFirstLane, SgprB32_ReadFirstLane}});
addRulesForIOpcs({amdgcn_permlane_idx_gen}, Standard)
.Div(S32, {{Vgpr32}, {IntrId, Vgpr32, SgprB32_ReadFirstLane}});
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 8b89366f89c5a..2de3dbd25d52a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -7859,6 +7859,10 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
IID == Intrinsic::amdgcn_permlanex16;
bool IsSetInactive = IID == Intrinsic::amdgcn_set_inactive ||
IID == Intrinsic::amdgcn_set_inactive_chain_arg;
+ bool IsPermlaneShuffle = IID == Intrinsic::amdgcn_permlane_bcast ||
+ IID == Intrinsic::amdgcn_permlane_up ||
+ IID == Intrinsic::amdgcn_permlane_down ||
+ IID == Intrinsic::amdgcn_permlane_xor;
SDLoc SL(N);
MVT IntVT = MVT::getIntegerVT(ValSize);
const GCNSubtarget *ST = TLI.getSubtarget();
@@ -7880,6 +7884,10 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
Operands.push_back(N->getOperand(4));
[[fallthrough]];
case Intrinsic::amdgcn_writelane:
+ case Intrinsic::amdgcn_permlane_bcast:
+ case Intrinsic::amdgcn_permlane_up:
+ case Intrinsic::amdgcn_permlane_down:
+ case Intrinsic::amdgcn_permlane_xor:
Operands.push_back(Src2);
[[fallthrough]];
case Intrinsic::amdgcn_readlane:
@@ -7913,10 +7921,12 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
SDValue Src1, Src2;
if (IID == Intrinsic::amdgcn_readlane || IID == Intrinsic::amdgcn_writelane ||
IID == Intrinsic::amdgcn_mov_dpp8 ||
- IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16) {
+ IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16 ||
+ IsPermlaneShuffle) {
Src1 = N->getOperand(2);
if (IID == Intrinsic::amdgcn_writelane ||
- IID == Intrinsic::amdgcn_update_dpp || IsPermLane16)
+ IID == Intrinsic::amdgcn_update_dpp || IsPermLane16 ||
+ IsPermlaneShuffle)
Src2 = N->getOperand(3);
}
@@ -8011,18 +8021,21 @@ static SDValue lowerLaneOp(const SITargetLowering &TLI, SDNode *N,
DAG.getConstant(EltIdx, SL, MVT::i32));
if (IID == Intrinsic::amdgcn_update_dpp || IsSetInactive ||
- IsPermLane16)
+ IsPermLane16) {
Src1SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src1,
DAG.getConstant(EltIdx, SL, MVT::i32));
- if (IID == Intrinsic::amdgcn_writelane)
+ Pieces.push_back(
+ createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT));
+ } else if (IID == Intrinsic::amdgcn_writelane) {
Src2SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SL, SubVecVT, Src2,
DAG.getConstant(EltIdx, SL, MVT::i32));
+ Pieces.push_back(
+ createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
+ } else {
+ Pieces.push_back(createLaneOp(Src0SubVec, Src1, Src2, SubVecVT));
+ }
- Pieces.push_back(
- IID == Intrinsic::amdgcn_update_dpp || IsSetInactive || IsPermLane16
- ? createLaneOp(Src0SubVec, Src1SubVec, Src2, SubVecVT)
- : createLaneOp(Src0SubVec, Src1, Src2SubVec, SubVecVT));
EltIdx += SubVecNumElt;
}
return DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, Pieces);
@@ -11144,6 +11157,10 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
case Intrinsic::amdgcn_set_inactive_chain_arg:
case Intrinsic::amdgcn_mov_dpp8:
case Intrinsic::amdgcn_update_dpp:
+ case Intrinsic::amdgcn_permlane_bcast:
+ case Intrinsic::amdgcn_permlane_up:
+ case Intrinsic::amdgcn_permlane_down:
+ case Intrinsic::amdgcn_permlane_xor:
return lowerLaneOp(*this, Op.getNode(), DAG);
case Intrinsic::amdgcn_dead: {
SmallVector<SDValue, 8> Poisons;
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index e78ecd1af9b45..d198a591c148a 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -1205,8 +1205,8 @@ class PermlaneVarPat<SDPatternOperator permlane,
>;
class PermlaneNoDppPat3Src<SDPatternOperator permlane,
- Instruction inst> : GCNPat<
- (permlane i32:$src0, i32:$src1, i32:$src2),
+ Instruction inst, ValueType vt> : GCNPat<
+ (vt (permlane vt:$src0, i32:$src1, i32:$src2)),
(inst VGPR_32:$src0, SCSrc_b32:$src1, SCSrc_b32:$src2)
>;
@@ -1611,10 +1611,12 @@ let SubtargetPredicate = isGFX1250Plus, WaveSizePredicate = isWave32 in {
defm V_PERMLANE_IDX_GEN_B32 : VOP3Inst<"v_permlane_idx_gen_b32", VOP3_PERMLANE_NOOPSEL_Profile<VOP_I32_I32_I32>>;
} // End isConvergent = 1
- def : PermlaneNoDppPat3Src<int_amdgcn_permlane_bcast, V_PERMLANE_BCAST_B32_e64>;
- def : PermlaneNoDppPat3Src<int_amdgcn_permlane_up, V_PERMLANE_UP_B32_e64>;
- def : PermlaneNoDppPat3Src<int_amdgcn_permlane_down, V_PERMLANE_DOWN_B32_e64>;
- def : PermlaneNoDppPat3Src<int_amdgcn_permlane_xor, V_PERMLANE_XOR_B32_e64>;
+ foreach vt = Reg32Types.types in {
+ def : PermlaneNoDppPat3Src<int_amdgcn_permlane_bcast, V_PERMLANE_BCAST_B32_e64, vt>;
+ def : PermlaneNoDppPat3Src<int_amdgcn_permlane_up, V_PERMLANE_UP_B32_e64, vt>;
+ def : PermlaneNoDppPat3Src<int_amdgcn_permlane_down, V_PERMLANE_DOWN_B32_e64, vt>;
+ def : PermlaneNoDppPat3Src<int_amdgcn_permlane_xor, V_PERMLANE_XOR_B32_e64, vt>;
+ }
def : PermlaneNoDppPat2Src<int_amdgcn_permlane_idx_gen, V_PERMLANE_IDX_GEN_B32_e64>;
} // End SubtargetPredicate = isGFX1250Plus, WaveSizePredicate = isWave32
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
index 68663ae820b57..9c57f1f2e5367 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/intrinsics.ll
@@ -802,28 +802,28 @@ define amdgpu_kernel void @v_permlane32_swap(ptr addrspace(1) %out, i32 %src0, i
ret void
}
-; CHECK: DIVERGENT: %result = call i32 @llvm.amdgcn.permlane.bcast(i32 %src0, i32 %src1, i32 %src2)
+; CHECK: DIVERGENT: %result = call i32 @llvm.amdgcn.permlane.bcast.i32(i32 %src0, i32 %src1, i32 %src2)
define amdgpu_kernel void @v_permlane_bcast_b32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
%result= call i32 @llvm.amdgcn.permlane.bcast(i32 %src0, i32 %src1, i32 %src2)
store i32 %result, ptr addrspace(1) %out
ret void
}
-; CHECK: DIVERGENT: %result = call i32 @llvm.amdgcn.permlane.up(i32 %src0, i32 %src1, i32 %src2)
+; CHECK: DIVERGENT: %result = call i32 @llvm.amdgcn.permlane.up.i32(i32 %src0, i32 %src1, i32 %src2)
define amdgpu_kernel void @v_permlane_up_b32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
%result= call i32 @llvm.amdgcn.permlane.up(i32 %src0, i32 %src1, i32 %src2)
store i32 %result, ptr addrspace(1) %out
ret void
}
-; CHECK: DIVERGENT: %result = call i32 @llvm.amdgcn.permlane.down(i32 %src0, i32 %src1, i32 %src2)
+; CHECK: DIVERGENT: %result = call i32 @llvm.amdgcn.permlane.down.i32(i32 %src0, i32 %src1, i32 %src2)
define amdgpu_kernel void @v_permlane_down_b32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
%result= call i32 @llvm.amdgcn.permlane.down(i32 %src0, i32 %src1, i32 %src2)
store i32 %result, ptr addrspace(1) %out
ret void
}
-; CHECK: DIVERGENT: %result = call i32 @llvm.amdgcn.permlane.xor(i32 %src0, i32 %src1, i32 %src2)
+; CHECK: DIVERGENT: %result = call i32 @llvm.amdgcn.permlane.xor.i32(i32 %src0, i32 %src1, i32 %src2)
define amdgpu_kernel void @v_permlane_xor_b32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
%result= call i32 @llvm.amdgcn.permlane.xor(i32 %src0, i32 %src1, i32 %src2)
store i32 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.gfx1250.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.gfx1250.ll
index 72a14536bebd4..7a11d4739f91b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane.gfx1250.ll
@@ -94,62 +94,62 @@ define amdgpu_kernel void @v_permlane_bcast_b32_vvv(ptr addrspace(1) %out, i32 %
ret void
}
-define amdgpu_kernel void @v_permlane_down_b32_vss(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX1250-LABEL: v_permlane_down_b32_vss:
+define amdgpu_kernel void @v_permlane_bcast_f32_vss(ptr addrspace(1) %out, float %src0, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_bcast_f32_vss:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_permlane_down_b32 v0, v0, s3, s6
+; GFX1250-NEXT: v_permlane_bcast_b32 v0, v0, s3, s6
; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
- %v = call i32 @llvm.amdgcn.permlane.down(i32 %src0, i32 %src1, i32 %src2)
- store i32 %v, ptr addrspace(1) %out
+ %v = call float @llvm.amdgcn.permlane.bcast(float %src0, i32 %src1, i32 %src2)
+ store float %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane_down_b32_vii(ptr addrspace(1) %out, i32 %src0) {
-; GFX1250-LABEL: v_permlane_down_b32_vii:
+define amdgpu_kernel void @v_permlane_bcast_f32_vii(ptr addrspace(1) %out, float %src0) {
+; GFX1250-LABEL: v_permlane_bcast_f32_vii:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_permlane_down_b32 v0, v0, 1, 2
+; GFX1250-NEXT: v_permlane_bcast_b32 v0, v0, 1, 2
; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
- %v = call i32 @llvm.amdgcn.permlane.down(i32 %src0, i32 1, i32 2)
- store i32 %v, ptr addrspace(1) %out
+ %v = call float @llvm.amdgcn.permlane.bcast(float %src0, i32 1, i32 2)
+ store float %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane_down_b32_vll(ptr addrspace(1) %out, i32 %src0) {
-; GFX1250-LABEL: v_permlane_down_b32_vll:
+define amdgpu_kernel void @v_permlane_bcast_f32_vll(ptr addrspace(1) %out, float %src0) {
+; GFX1250-LABEL: v_permlane_bcast_f32_vll:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-NEXT: s_movk_i32 s2, 0x64
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_permlane_down_b32 v0, v0, s2, 0x66
+; GFX1250-NEXT: v_permlane_bcast_b32 v0, v0, s2, 0x66
; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
- %v = call i32 @llvm.amdgcn.permlane.down(i32 %src0, i32 100, i32 102)
- store i32 %v, ptr addrspace(1) %out
+ %v = call float @llvm.amdgcn.permlane.bcast(float %src0, i32 100, i32 102)
+ store float %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane_down_b32_vvv(ptr addrspace(1) %out, i32 %src0) {
-; GFX1250-SDAG-LABEL: v_permlane_down_b32_vvv:
+define amdgpu_kernel void @v_permlane_bcast_f32_vvv(ptr addrspace(1) %out, float %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_f32_vvv:
; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
@@ -159,14 +159,14 @@ define amdgpu_kernel void @v_permlane_down_b32_vvv(ptr addrspace(1) %out, i32 %s
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v1, s3, s2
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v1, v1, s3, s2
; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1250-SDAG-NEXT: s_endpgm
;
-; GFX1250-GISEL-LABEL: v_permlane_down_b32_vvv:
+; GFX1250-GISEL-LABEL: v_permlane_bcast_f32_vvv:
; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
@@ -176,18 +176,638 @@ define amdgpu_kernel void @v_permlane_down_b32_vvv(ptr addrspace(1) %out, i32 %s
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, s3, s4
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v0, v0, s3, s4
; GFX1250-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidy = call i32 @llvm.amdgcn.workitem.id.y()
- %v = call i32 @llvm.amdgcn.permlane.down(i32 %src0, i32 %tidx, i32 %tidy)
- store i32 %v, ptr addrspace(1) %out
+ %v = call float @llvm.amdgcn.permlane.bcast(float %src0, i32 %tidx, i32 %tidy)
+ store float %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane_up_b32_vss(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX1250-LABEL: v_permlane_up_b32_vss:
+define amdgpu_kernel void @v_permlane_bcast_i64_vss(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_i64_vss:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v1, v0, s6, s7
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v0, v2, s6, s7
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_i64_vss:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v0, v0, s6, s7
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v1, v1, s6, s7
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.bcast(i64 %src0, i32 %src1, i32 %src2)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_bcast_i64_vii(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_i64_vii:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v1, v0, 1, 2
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v0, v2, 1, 2
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_i64_vii:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v0, v0, 1, 2
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v1, v1, 1, 2
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.bcast(i64 %src0, i32 1, i32 2)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_bcast_i64_vll(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_i64_vll:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v1, v0, s2, 0x66
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v0, v2, s2, 0x66
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_i64_vll:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v0, v0, s2, 0x66
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v1, v1, s2, 0x66
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.bcast(i64 %src0, i32 100, i32 102)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_bcast_i64_vvv(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_i64_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v1, v0, s4, s5
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v0, v2, s4, s5
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_i64_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v0, v0, s4, s5
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v1, v1, s4, s5
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call i64 @llvm.amdgcn.permlane.bcast(i64 %src0, i32 %tidx, i32 %tidy)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_bcast_f64_vss(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_f64_vss:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v1, v0, s6, s7
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v0, v2, s6, s7
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_f64_vss:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v0, v0, s6, s7
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v1, v1, s6, s7
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.bcast(double %src0, i32 %src1, i32 %src2)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_bcast_f64_vii(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_f64_vii:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v1, v0, 1, 2
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v0, v2, 1, 2
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_f64_vii:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v0, v0, 1, 2
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v1, v1, 1, 2
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.bcast(double %src0, i32 1, i32 2)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_bcast_f64_vll(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_f64_vll:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v1, v0, s2, 0x66
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v0, v2, s2, 0x66
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_f64_vll:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v0, v0, s2, 0x66
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v1, v1, s2, 0x66
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.bcast(double %src0, i32 100, i32 102)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_bcast_f64_vvv(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_f64_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v1, v0, s4, s5
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v0, v2, s4, s5
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_f64_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v0, v0, s4, s5
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v1, v1, s4, s5
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call double @llvm.amdgcn.permlane.bcast(double %src0, i32 %tidx, i32 %tidy)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+; does not work for GISEL
+;define void @v_permlane_bcast_bfloat(ptr addrspace(1) %out, bfloat %src, i32 %src1, i32 %src2) {
+; %v = call bfloat @llvm.amdgcn.permlane.bcast.bf16(bfloat %src, i32 %src1, i32 %src2)
+; store bfloat %v, ptr addrspace(1) %out, align 4
+; ret void
+;}
+
+define void @v_permlane_bcast_i16(ptr addrspace(1) %out, i16 %src, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_bcast_i16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %v = call i16 @llvm.amdgcn.permlane.bcast.i16(i16 %src, i32 %src1, i32 %src2)
+ store i16 %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_bcast_v2f16(ptr addrspace(1) %out, <2 x half> %src, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_bcast_v2f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x half> @llvm.amdgcn.permlane.bcast.v2f16(<2 x half> %src, i32 %src1, i32 %src2)
+ store <2 x half> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_bcast_v2f32(ptr addrspace(1) %out, <2 x float> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_v2f32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_v2f32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x float> @llvm.amdgcn.permlane.bcast.v2f32(<2 x float> %src, i32 %src1, i32 %src2)
+ store <2 x float> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_bcast_v7i32(ptr addrspace(1) %out, <7 x i32> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_v7i32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v9
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v10
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_v7i32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v9
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v10
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <7 x i32> @llvm.amdgcn.permlane.bcast.v7i32(<7 x i32> %src, i32 %src1, i32 %src2)
+ store <7 x i32> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_bcast_v8i16(ptr addrspace(1) %out, <8 x i16> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_v8i16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_v8i16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <8 x i16> @llvm.amdgcn.permlane.bcast.v8i16(<8 x i16> %src, i32 %src1, i32 %src2)
+ store <8 x i16> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_bcast_v2i64(ptr addrspace(1) %out, <2 x i64> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_v2i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_v2i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x i64> @llvm.amdgcn.permlane.bcast.v2i64(<2 x i64> %src, i32 %src1, i32 %src2)
+ store <2 x i64> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_bcast_v3i64(ptr addrspace(1) %out, <3 x i64> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_v3i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_v3i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <3 x i64> @llvm.amdgcn.permlane.bcast.v2i64(<3 x i64> %src, i32 %src1, i32 %src2)
+ store <3 x i64> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_bcast_v4f64(ptr addrspace(1) %out, <4 x double> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_v4f64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v10
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v11
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v9, v9, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_v4f64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v10
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v11
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v9, v9, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <4 x double> @llvm.amdgcn.permlane.bcast.v4f64(<4 x double> %src, i32 %src1, i32 %src2)
+ store <4 x double> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_bcast_v8f64(ptr addrspace(1) %out, <8 x double> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_bcast_v8f64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v18
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v19
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v17, v17, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v16, v16, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v15, v15, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v14, v14, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v13, v13, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v12, v12, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v11, v11, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v10, v10, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v9, v9, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x3
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_bcast_v8f64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v9, v9, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v10, v10, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v11, v11, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v12, v12, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v13, v13, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v14, v14, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v15, v15, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v16, v16, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_bcast_b32 v17, v17, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x3
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <8 x double> @llvm.amdgcn.permlane.bcast.v8f64(<8 x double> %src, i32 %src1, i32 %src2)
+ store <8 x double> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_b32_vss(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_down_b32_vss:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
@@ -196,32 +816,32 @@ define amdgpu_kernel void @v_permlane_up_b32_vss(ptr addrspace(1) %out, i32 %src
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_permlane_up_b32 v0, v0, s3, s6
+; GFX1250-NEXT: v_permlane_down_b32 v0, v0, s3, s6
; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
- %v = call i32 @llvm.amdgcn.permlane.up(i32 %src0, i32 %src1, i32 %src2)
+ %v = call i32 @llvm.amdgcn.permlane.down(i32 %src0, i32 %src1, i32 %src2)
store i32 %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane_up_b32_vii(ptr addrspace(1) %out, i32 %src0) {
-; GFX1250-LABEL: v_permlane_up_b32_vii:
+define amdgpu_kernel void @v_permlane_down_b32_vii(ptr addrspace(1) %out, i32 %src0) {
+; GFX1250-LABEL: v_permlane_down_b32_vii:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_permlane_up_b32 v0, v0, 1, 2
+; GFX1250-NEXT: v_permlane_down_b32 v0, v0, 1, 2
; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
- %v = call i32 @llvm.amdgcn.permlane.up(i32 %src0, i32 1, i32 2)
+ %v = call i32 @llvm.amdgcn.permlane.down(i32 %src0, i32 1, i32 2)
store i32 %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane_up_b32_vll(ptr addrspace(1) %out, i32 %src0) {
-; GFX1250-LABEL: v_permlane_up_b32_vll:
+define amdgpu_kernel void @v_permlane_down_b32_vll(ptr addrspace(1) %out, i32 %src0) {
+; GFX1250-LABEL: v_permlane_down_b32_vll:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
@@ -229,16 +849,16 @@ define amdgpu_kernel void @v_permlane_up_b32_vll(ptr addrspace(1) %out, i32 %src
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-NEXT: s_movk_i32 s2, 0x64
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_permlane_up_b32 v0, v0, s2, 0x66
+; GFX1250-NEXT: v_permlane_down_b32 v0, v0, s2, 0x66
; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
- %v = call i32 @llvm.amdgcn.permlane.up(i32 %src0, i32 100, i32 102)
+ %v = call i32 @llvm.amdgcn.permlane.down(i32 %src0, i32 100, i32 102)
store i32 %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane_up_b32_vvv(ptr addrspace(1) %out, i32 %src0) {
-; GFX1250-SDAG-LABEL: v_permlane_up_b32_vvv:
+define amdgpu_kernel void @v_permlane_down_b32_vvv(ptr addrspace(1) %out, i32 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_down_b32_vvv:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
@@ -251,11 +871,11 @@ define amdgpu_kernel void @v_permlane_up_b32_vvv(ptr addrspace(1) %out, i32 %src
; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v0
; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v1, s3, s2
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v1, s3, s2
; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1250-SDAG-NEXT: s_endpgm
;
-; GFX1250-GISEL-LABEL: v_permlane_up_b32_vvv:
+; GFX1250-GISEL-LABEL: v_permlane_down_b32_vvv:
; GFX1250-GISEL: ; %bb.0:
; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
@@ -268,35 +888,1551 @@ define amdgpu_kernel void @v_permlane_up_b32_vvv(ptr addrspace(1) %out, i32 %src
; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, s3, s4
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, s3, s4
; GFX1250-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-GISEL-NEXT: s_endpgm
%tidx = call i32 @llvm.amdgcn.workitem.id.x()
%tidy = call i32 @llvm.amdgcn.workitem.id.y()
- %v = call i32 @llvm.amdgcn.permlane.up(i32 %src0, i32 %tidx, i32 %tidy)
+ %v = call i32 @llvm.amdgcn.permlane.down(i32 %src0, i32 %tidx, i32 %tidy)
store i32 %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane_xor_b32_vss(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
-; GFX1250-LABEL: v_permlane_xor_b32_vss:
+define amdgpu_kernel void @v_permlane_down_f32_vss(ptr addrspace(1) %out, float %src0, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_down_f32_vss:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_permlane_xor_b32 v0, v0, s3, s6
+; GFX1250-NEXT: v_permlane_down_b32 v0, v0, s3, s6
; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
- %v = call i32 @llvm.amdgcn.permlane.xor(i32 %src0, i32 %src1, i32 %src2)
- store i32 %v, ptr addrspace(1) %out
+ %v = call float @llvm.amdgcn.permlane.down(float %src0, i32 %src1, i32 %src2)
+ store float %v, ptr addrspace(1) %out
ret void
}
-define amdgpu_kernel void @v_permlane_xor_b32_vii(ptr addrspace(1) %out, i32 %src0) {
+define amdgpu_kernel void @v_permlane_down_f32_vii(ptr addrspace(1) %out, float %src0) {
+; GFX1250-LABEL: v_permlane_down_f32_vii:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_down_b32 v0, v0, 1, 2
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call float @llvm.amdgcn.permlane.down(float %src0, i32 1, i32 2)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_f32_vll(ptr addrspace(1) %out, float %src0) {
+; GFX1250-LABEL: v_permlane_down_f32_vll:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: v_permlane_down_b32 v0, v0, s2, 0x66
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call float @llvm.amdgcn.permlane.down(float %src0, i32 100, i32 102)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_f32_vvv(ptr addrspace(1) %out, float %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_down_f32_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v1, s3, s2
+; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_f32_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, s3, s4
+; GFX1250-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call float @llvm.amdgcn.permlane.down(float %src0, i32 %tidx, i32 %tidy)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_i64_vss(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_down_i64_vss:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v0, s6, s7
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v0, v2, s6, s7
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_i64_vss:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, s6, s7
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v1, v1, s6, s7
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.down(i64 %src0, i32 %src1, i32 %src2)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_i64_vii(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_down_i64_vii:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v0, 1, 2
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v0, v2, 1, 2
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_i64_vii:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, 1, 2
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v1, v1, 1, 2
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.down(i64 %src0, i32 1, i32 2)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_i64_vll(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_down_i64_vll:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v0, s2, 0x66
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v0, v2, s2, 0x66
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_i64_vll:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, s2, 0x66
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v1, v1, s2, 0x66
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.down(i64 %src0, i32 100, i32 102)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_i64_vvv(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_down_i64_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v0, s4, s5
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v0, v2, s4, s5
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_i64_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, s4, s5
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v1, v1, s4, s5
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call i64 @llvm.amdgcn.permlane.down(i64 %src0, i32 %tidx, i32 %tidy)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_f64_vss(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_down_f64_vss:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v0, s6, s7
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v0, v2, s6, s7
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_f64_vss:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, s6, s7
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v1, v1, s6, s7
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.down(double %src0, i32 %src1, i32 %src2)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_f64_vii(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_down_f64_vii:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v0, 1, 2
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v0, v2, 1, 2
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_f64_vii:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, 1, 2
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v1, v1, 1, 2
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.down(double %src0, i32 1, i32 2)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_f64_vll(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_down_f64_vll:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v0, s2, 0x66
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v0, v2, s2, 0x66
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_f64_vll:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, s2, 0x66
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v1, v1, s2, 0x66
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.down(double %src0, i32 100, i32 102)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_down_f64_vvv(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_down_f64_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v1, v0, s4, s5
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v0, v2, s4, s5
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_f64_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v0, v0, s4, s5
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v1, v1, s4, s5
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call double @llvm.amdgcn.permlane.down(double %src0, i32 %tidx, i32 %tidy)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+; does not work for GISEL
+;define void @v_permlane_down_bfloat(ptr addrspace(1) %out, bfloat %src, i32 %src1, i32 %src2) {
+; %v = call bfloat @llvm.amdgcn.permlane.down.bf16(bfloat %src, i32 %src1, i32 %src2)
+; store bfloat %v, ptr addrspace(1) %out, align 4
+; ret void
+;}
+
+define void @v_permlane_down_i16(ptr addrspace(1) %out, i16 %src, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_down_i16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %v = call i16 @llvm.amdgcn.permlane.down.i16(i16 %src, i32 %src1, i32 %src2)
+ store i16 %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_down_v2f16(ptr addrspace(1) %out, <2 x half> %src, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_down_v2f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x half> @llvm.amdgcn.permlane.down.v2f16(<2 x half> %src, i32 %src1, i32 %src2)
+ store <2 x half> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_down_v2f32(ptr addrspace(1) %out, <2 x float> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_down_v2f32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_v2f32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x float> @llvm.amdgcn.permlane.down.v2f32(<2 x float> %src, i32 %src1, i32 %src2)
+ store <2 x float> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_down_v7i32(ptr addrspace(1) %out, <7 x i32> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_down_v7i32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v9
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v10
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_v7i32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v9
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v10
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <7 x i32> @llvm.amdgcn.permlane.down.v7i32(<7 x i32> %src, i32 %src1, i32 %src2)
+ store <7 x i32> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_down_v8i16(ptr addrspace(1) %out, <8 x i16> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_down_v8i16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_v8i16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <8 x i16> @llvm.amdgcn.permlane.down.v8i16(<8 x i16> %src, i32 %src1, i32 %src2)
+ store <8 x i16> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_down_v2i64(ptr addrspace(1) %out, <2 x i64> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_down_v2i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_v2i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x i64> @llvm.amdgcn.permlane.down.v2i64(<2 x i64> %src, i32 %src1, i32 %src2)
+ store <2 x i64> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_down_v3i64(ptr addrspace(1) %out, <3 x i64> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_down_v3i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_v3i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <3 x i64> @llvm.amdgcn.permlane.down.v2i64(<3 x i64> %src, i32 %src1, i32 %src2)
+ store <3 x i64> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_down_v4f64(ptr addrspace(1) %out, <4 x double> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_down_v4f64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v10
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v11
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v9, v9, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_v4f64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v10
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v11
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v9, v9, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <4 x double> @llvm.amdgcn.permlane.down.v4f64(<4 x double> %src, i32 %src1, i32 %src2)
+ store <4 x double> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_down_v8f64(ptr addrspace(1) %out, <8 x double> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_down_v8f64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v18
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v19
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v17, v17, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v16, v16, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v15, v15, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v14, v14, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v13, v13, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v12, v12, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v11, v11, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v10, v10, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v9, v9, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x3
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_down_v8f64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v9, v9, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v10, v10, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v11, v11, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v12, v12, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v13, v13, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v14, v14, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v15, v15, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v16, v16, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_down_b32 v17, v17, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x3
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <8 x double> @llvm.amdgcn.permlane.down.v8f64(<8 x double> %src, i32 %src1, i32 %src2)
+ store <8 x double> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_b32_vss(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_up_b32_vss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_up_b32 v0, v0, s3, s6
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call i32 @llvm.amdgcn.permlane.up(i32 %src0, i32 %src1, i32 %src2)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_b32_vii(ptr addrspace(1) %out, i32 %src0) {
+; GFX1250-LABEL: v_permlane_up_b32_vii:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_up_b32 v0, v0, 1, 2
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call i32 @llvm.amdgcn.permlane.up(i32 %src0, i32 1, i32 2)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_b32_vll(ptr addrspace(1) %out, i32 %src0) {
+; GFX1250-LABEL: v_permlane_up_b32_vll:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: v_permlane_up_b32 v0, v0, s2, 0x66
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call i32 @llvm.amdgcn.permlane.up(i32 %src0, i32 100, i32 102)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_b32_vvv(ptr addrspace(1) %out, i32 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_up_b32_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v1, s3, s2
+; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_b32_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, s3, s4
+; GFX1250-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call i32 @llvm.amdgcn.permlane.up(i32 %src0, i32 %tidx, i32 %tidy)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_f32_vss(ptr addrspace(1) %out, float %src0, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_up_f32_vss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_up_b32 v0, v0, s3, s6
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call float @llvm.amdgcn.permlane.up(float %src0, i32 %src1, i32 %src2)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_f32_vii(ptr addrspace(1) %out, float %src0) {
+; GFX1250-LABEL: v_permlane_up_f32_vii:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_up_b32 v0, v0, 1, 2
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call float @llvm.amdgcn.permlane.up(float %src0, i32 1, i32 2)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_f32_vll(ptr addrspace(1) %out, float %src0) {
+; GFX1250-LABEL: v_permlane_up_f32_vll:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: v_permlane_up_b32 v0, v0, s2, 0x66
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call float @llvm.amdgcn.permlane.up(float %src0, i32 100, i32 102)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_f32_vvv(ptr addrspace(1) %out, float %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_up_f32_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v1, s3, s2
+; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_f32_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, s3, s4
+; GFX1250-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call float @llvm.amdgcn.permlane.up(float %src0, i32 %tidx, i32 %tidy)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_i64_vss(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_up_i64_vss:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v0, s6, s7
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v0, v2, s6, s7
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_i64_vss:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, s6, s7
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v1, v1, s6, s7
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.up(i64 %src0, i32 %src1, i32 %src2)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_i64_vii(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_up_i64_vii:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v0, 1, 2
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v0, v2, 1, 2
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_i64_vii:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, 1, 2
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v1, v1, 1, 2
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.up(i64 %src0, i32 1, i32 2)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_i64_vll(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_up_i64_vll:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v0, s2, 0x66
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v0, v2, s2, 0x66
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_i64_vll:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, s2, 0x66
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v1, v1, s2, 0x66
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.up(i64 %src0, i32 100, i32 102)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_i64_vvv(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_up_i64_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v0, s4, s5
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v0, v2, s4, s5
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_i64_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, s4, s5
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v1, v1, s4, s5
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call i64 @llvm.amdgcn.permlane.up(i64 %src0, i32 %tidx, i32 %tidy)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_f64_vss(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_up_f64_vss:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v0, s6, s7
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v0, v2, s6, s7
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_f64_vss:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, s6, s7
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v1, v1, s6, s7
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.up(double %src0, i32 %src1, i32 %src2)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_f64_vii(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_up_f64_vii:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v0, 1, 2
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v0, v2, 1, 2
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_f64_vii:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, 1, 2
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v1, v1, 1, 2
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.up(double %src0, i32 1, i32 2)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_f64_vll(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_up_f64_vll:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v0, s2, 0x66
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v0, v2, s2, 0x66
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_f64_vll:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, s2, 0x66
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v1, v1, s2, 0x66
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.up(double %src0, i32 100, i32 102)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_up_f64_vvv(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_up_f64_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v1, v0, s4, s5
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v0, v2, s4, s5
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_f64_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v0, v0, s4, s5
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v1, v1, s4, s5
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call double @llvm.amdgcn.permlane.up(double %src0, i32 %tidx, i32 %tidy)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+; does not work for GISEL
+;define void @v_permlane_up_bfloat(ptr addrspace(1) %out, bfloat %src, i32 %src1, i32 %src2) {
+; %v = call bfloat @llvm.amdgcn.permlane.up.bf16(bfloat %src, i32 %src1, i32 %src2)
+; store bfloat %v, ptr addrspace(1) %out, align 4
+; ret void
+;}
+
+define void @v_permlane_up_i16(ptr addrspace(1) %out, i16 %src, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_up_i16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %v = call i16 @llvm.amdgcn.permlane.up.i16(i16 %src, i32 %src1, i32 %src2)
+ store i16 %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_up_v2f16(ptr addrspace(1) %out, <2 x half> %src, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_up_v2f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x half> @llvm.amdgcn.permlane.up.v2f16(<2 x half> %src, i32 %src1, i32 %src2)
+ store <2 x half> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_up_v2f32(ptr addrspace(1) %out, <2 x float> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_up_v2f32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_v2f32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x float> @llvm.amdgcn.permlane.up.v2f32(<2 x float> %src, i32 %src1, i32 %src2)
+ store <2 x float> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_up_v7i32(ptr addrspace(1) %out, <7 x i32> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_up_v7i32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v9
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v10
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_v7i32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v9
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v10
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <7 x i32> @llvm.amdgcn.permlane.up.v7i32(<7 x i32> %src, i32 %src1, i32 %src2)
+ store <7 x i32> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_up_v8i16(ptr addrspace(1) %out, <8 x i16> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_up_v8i16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_v8i16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <8 x i16> @llvm.amdgcn.permlane.up.v8i16(<8 x i16> %src, i32 %src1, i32 %src2)
+ store <8 x i16> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_up_v2i64(ptr addrspace(1) %out, <2 x i64> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_up_v2i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_v2i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x i64> @llvm.amdgcn.permlane.up.v2i64(<2 x i64> %src, i32 %src1, i32 %src2)
+ store <2 x i64> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_up_v3i64(ptr addrspace(1) %out, <3 x i64> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_up_v3i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_v3i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <3 x i64> @llvm.amdgcn.permlane.up.v2i64(<3 x i64> %src, i32 %src1, i32 %src2)
+ store <3 x i64> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_up_v4f64(ptr addrspace(1) %out, <4 x double> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_up_v4f64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v10
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v11
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v9, v9, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_v4f64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v10
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v11
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v9, v9, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <4 x double> @llvm.amdgcn.permlane.up.v4f64(<4 x double> %src, i32 %src1, i32 %src2)
+ store <4 x double> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_up_v8f64(ptr addrspace(1) %out, <8 x double> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_up_v8f64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v18
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v19
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v17, v17, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v16, v16, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v15, v15, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v14, v14, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v13, v13, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v12, v12, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v11, v11, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v10, v10, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v9, v9, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x3
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_up_v8f64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v9, v9, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v10, v10, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v11, v11, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v12, v12, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v13, v13, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v14, v14, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v15, v15, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v16, v16, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_up_b32 v17, v17, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x3
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <8 x double> @llvm.amdgcn.permlane.up.v8f64(<8 x double> %src, i32 %src1, i32 %src2)
+ store <8 x double> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_b32_vss(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_xor_b32_vss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_xor_b32 v0, v0, s3, s6
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call i32 @llvm.amdgcn.permlane.xor(i32 %src0, i32 %src1, i32 %src2)
+ store i32 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_b32_vii(ptr addrspace(1) %out, i32 %src0) {
; GFX1250-LABEL: v_permlane_xor_b32_vii:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -370,6 +2506,718 @@ define amdgpu_kernel void @v_permlane_xor_b32_vvv(ptr addrspace(1) %out, i32 %sr
ret void
}
+define amdgpu_kernel void @v_permlane_xor_f32_vss(ptr addrspace(1) %out, float %src0, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_xor_f32_vss:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_xor_b32 v0, v0, s3, s6
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call float @llvm.amdgcn.permlane.xor(float %src0, i32 %src1, i32 %src2)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_f32_vii(ptr addrspace(1) %out, float %src0) {
+; GFX1250-LABEL: v_permlane_xor_f32_vii:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_xor_b32 v0, v0, 1, 2
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call float @llvm.amdgcn.permlane.xor(float %src0, i32 1, i32 2)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_f32_vll(ptr addrspace(1) %out, float %src0) {
+; GFX1250-LABEL: v_permlane_xor_f32_vll:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: v_permlane_xor_b32 v0, v0, s2, 0x66
+; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-NEXT: s_endpgm
+ %v = call float @llvm.amdgcn.permlane.xor(float %src0, i32 100, i32 102)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_f32_vvv(ptr addrspace(1) %out, float %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_f32_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v1, v1, s3, s2
+; GFX1250-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_f32_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v0, v0, s3, s4
+; GFX1250-GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call float @llvm.amdgcn.permlane.xor(float %src0, i32 %tidx, i32 %tidy)
+ store float %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_i64_vss(ptr addrspace(1) %out, i64 %src0, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_i64_vss:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v1, v0, s6, s7
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v0, v2, s6, s7
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_i64_vss:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v0, v0, s6, s7
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v1, v1, s6, s7
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.xor(i64 %src0, i32 %src1, i32 %src2)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_i64_vii(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_i64_vii:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v1, v0, 1, 2
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v0, v2, 1, 2
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_i64_vii:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v0, v0, 1, 2
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v1, v1, 1, 2
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.xor(i64 %src0, i32 1, i32 2)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_i64_vll(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_i64_vll:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v1, v0, s2, 0x66
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v0, v2, s2, 0x66
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_i64_vll:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v0, v0, s2, 0x66
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v1, v1, s2, 0x66
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call i64 @llvm.amdgcn.permlane.xor(i64 %src0, i32 100, i32 102)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_i64_vvv(ptr addrspace(1) %out, i64 %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_i64_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v1, v0, s4, s5
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v0, v2, s4, s5
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_i64_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v0, v0, s4, s5
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v1, v1, s4, s5
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call i64 @llvm.amdgcn.permlane.xor(i64 %src0, i32 %tidx, i32 %tidy)
+ store i64 %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_f64_vss(ptr addrspace(1) %out, double %src0, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_f64_vss:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v1, v0, s6, s7
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v0, v2, s6, s7
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_f64_vss:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v0, v0, s6, s7
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v1, v1, s6, s7
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.xor(double %src0, i32 %src1, i32 %src2)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_f64_vii(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_f64_vii:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v1, v0, 1, 2
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v0, v2, 1, 2
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_f64_vii:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v0, v0, 1, 2
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v1, v1, 1, 2
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.xor(double %src0, i32 1, i32 2)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_f64_vll(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_f64_vll:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-SDAG-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v1, v0, s2, 0x66
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v0, v2, s2, 0x66
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_f64_vll:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-GISEL-NEXT: s_movk_i32 s2, 0x64
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v0, v0, s2, 0x66
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v1, v1, s2, 0x66
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %v = call double @llvm.amdgcn.permlane.xor(double %src0, i32 100, i32 102)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @v_permlane_xor_f64_vvv(ptr addrspace(1) %out, double %src0) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_f64_vvv:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-SDAG-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-SDAG-NEXT: s_wait_xcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v2, s2
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v1, v0, s4, s5
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v0, v2, s4, s5
+; GFX1250-SDAG-NEXT: global_store_b64 v3, v[0:1], s[0:1]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_f64_vvv:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1250-GISEL-NEXT: v_bfe_u32 v0, v0, 10, 10
+; GFX1250-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-GISEL-NEXT: s_wait_xcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v0, v0, s4, s5
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v1, v1, s4, s5
+; GFX1250-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX1250-GISEL-NEXT: s_endpgm
+ %tidx = call i32 @llvm.amdgcn.workitem.id.x()
+ %tidy = call i32 @llvm.amdgcn.workitem.id.y()
+ %v = call double @llvm.amdgcn.permlane.xor(double %src0, i32 %tidx, i32 %tidy)
+ store double %v, ptr addrspace(1) %out
+ ret void
+}
+
+; does not work for GISEL
+;define void @v_permlane_xor_bfloat(ptr addrspace(1) %out, bfloat %src, i32 %src1, i32 %src2) {
+; %v = call bfloat @llvm.amdgcn.permlane.xor.bf16(bfloat %src, i32 %src1, i32 %src2)
+; store bfloat %v, ptr addrspace(1) %out, align 4
+; ret void
+;}
+
+define void @v_permlane_xor_i16(ptr addrspace(1) %out, i16 %src, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_xor_i16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %v = call i16 @llvm.amdgcn.permlane.xor.i16(i16 %src, i32 %src1, i32 %src2)
+ store i16 %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_xor_v2f16(ptr addrspace(1) %out, <2 x half> %src, i32 %src1, i32 %src2) {
+; GFX1250-LABEL: v_permlane_xor_v2f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_readfirstlane_b32 s0, v3
+; GFX1250-NEXT: v_readfirstlane_b32 s1, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x half> @llvm.amdgcn.permlane.xor.v2f16(<2 x half> %src, i32 %src1, i32 %src2)
+ store <2 x half> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_xor_v2f32(ptr addrspace(1) %out, <2 x float> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_v2f32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_v2f32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v4
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x float> @llvm.amdgcn.permlane.xor.v2f32(<2 x float> %src, i32 %src1, i32 %src2)
+ store <2 x float> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_xor_v7i32(ptr addrspace(1) %out, <7 x i32> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_v7i32:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v9
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v10
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_v7i32:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v9
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v10
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b96 v[0:1], v[6:8], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <7 x i32> @llvm.amdgcn.permlane.xor.v7i32(<7 x i32> %src, i32 %src1, i32 %src2)
+ store <7 x i32> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_xor_v8i16(ptr addrspace(1) %out, <8 x i16> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_v8i16:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_v8i16:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <8 x i16> @llvm.amdgcn.permlane.xor.v8i16(<8 x i16> %src, i32 %src1, i32 %src2)
+ store <8 x i16> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_xor_v2i64(ptr addrspace(1) %out, <2 x i64> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_v2i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_v2i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v6
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v7
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <2 x i64> @llvm.amdgcn.permlane.xor.v2i64(<2 x i64> %src, i32 %src1, i32 %src2)
+ store <2 x i64> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_xor_v3i64(ptr addrspace(1) %out, <3 x i64> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_v3i64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_v3i64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v8
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v9
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b64 v[0:1], v[6:7], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <3 x i64> @llvm.amdgcn.permlane.xor.v2i64(<3 x i64> %src, i32 %src1, i32 %src2)
+ store <3 x i64> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_xor_v4f64(ptr addrspace(1) %out, <4 x double> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_v4f64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v10
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v11
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v9, v9, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x1
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_v4f64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v10
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v11
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v9, v9, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x1
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <4 x double> @llvm.amdgcn.permlane.xor.v4f64(<4 x double> %src, i32 %src1, i32 %src2)
+ store <4 x double> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @v_permlane_xor_v8f64(ptr addrspace(1) %out, <8 x double> %src, i32 %src1, i32 %src2) {
+; GFX1250-SDAG-LABEL: v_permlane_xor_v8f64:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s0, v18
+; GFX1250-SDAG-NEXT: v_readfirstlane_b32 s1, v19
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v17, v17, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v16, v16, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v15, v15, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v14, v14, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v13, v13, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v12, v12, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v11, v11, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v10, v10, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v9, v9, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v8, v8, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v7, v7, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v6, v6, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-SDAG-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-SDAG-NEXT: s_clause 0x3
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-SDAG-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-SDAG-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250-GISEL-LABEL: v_permlane_xor_v8f64:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s0, v18
+; GFX1250-GISEL-NEXT: v_readfirstlane_b32 s1, v19
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v2, v2, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v3, v3, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v4, v4, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v5, v5, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v6, v6, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v7, v7, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v8, v8, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v9, v9, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v10, v10, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v11, v11, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v12, v12, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v13, v13, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v14, v14, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v15, v15, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v16, v16, s0, s1
+; GFX1250-GISEL-NEXT: v_permlane_xor_b32 v17, v17, s0, s1
+; GFX1250-GISEL-NEXT: s_clause 0x3
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[10:13], off offset:32
+; GFX1250-GISEL-NEXT: global_store_b128 v[0:1], v[14:17], off offset:48
+; GFX1250-GISEL-NEXT: s_set_pc_i64 s[30:31]
+ %v = call <8 x double> @llvm.amdgcn.permlane.xor.v8f64(<8 x double> %src, i32 %src1, i32 %src2)
+ store <8 x double> %v, ptr addrspace(1) %out, align 4
+ ret void
+}
+
define amdgpu_kernel void @v_permlane_idx_gen_b32_vs(ptr addrspace(1) %out, i32 %src0, i32 %src1) {
; GFX1250-LABEL: v_permlane_idx_gen_b32_vs:
; GFX1250: ; %bb.0:
More information about the llvm-commits
mailing list