[llvm] [AMDGPU] Add additional optimization cases for llvm.amdgcn.wave.shuffle intrinsic (PR #182357)
Domenic Nutile via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 19 11:58:37 PST 2026
https://github.com/saxlungs created https://github.com/llvm/llvm-project/pull/182357
New optimizations include self-select optimizing to just passing through Val, and the DPP16 RowXMask, RowMirror, and RowHalfMirror modes
>From 90c4f55526a0823dfae4310f024b6831725f822f Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Thu, 19 Feb 2026 14:55:52 -0500
Subject: [PATCH] [AMDGPU] Add additional optimization cases for
llvm.amdgcn.wave.shuffle intrinsic
New optimizations include self-select optimizing to just passing through Val, and the DPP16 RowXMask, RowMirror, and RowHalfMirror modes
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 175 +-
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 12 +
.../AMDGPU/llvm.amdgcn.wave.shuffle.ll | 1840 +++++++++++++++--
3 files changed, 1790 insertions(+), 237 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 0ebe69de56fa9..9f08922af0611 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -554,8 +554,8 @@ static CallInst *rewriteCall(IRBuilderBase &B, CallInst &Old,
}
// Return true for sequences of instructions that effectively assign
-// each lane to its thread ID
-static bool isThreadID(const GCNSubtarget &ST, Value *V) {
+// each lane to its lane ID
+static bool isLaneID(const GCNSubtarget &ST, Value *V) {
// Case 1:
// wave32: mbcnt_lo(-1, 0)
// wave64: mbcnt_hi(-1, mbcnt_lo(-1, 0))
@@ -573,63 +573,145 @@ static bool isThreadID(const GCNSubtarget &ST, Value *V) {
}
// Attempt to capture situations where the index argument matches
-// a DPP pattern, and convert to a DPP-based mov
+// a DPP pattern, and convert to an amdgcn_update_dpp call
static std::optional<Instruction *>
-tryWaveShuffleDPP(const GCNSubtarget &ST, InstCombiner &IC, IntrinsicInst &II) {
+tryWaveShuffleDpp(const GCNSubtarget &ST, InstCombiner &IC, IntrinsicInst &II) {
Value *Val = II.getArgOperand(0);
Value *Idx = II.getArgOperand(1);
auto &B = IC.Builder;
- // DPP16 Row Share requires known wave size, architecture support
- if (!ST.isWaveSizeKnown() || !ST.hasDPPRowShare())
+ Value *Lid;
+
+ bool CanDppOpt = false;
+ uint64_t DppMode;
+
+ // Currently, all implemented DPP optimizations require known wave size
+ if (!ST.isWaveSizeKnown())
return std::nullopt;
- Value *Tid;
- uint64_t Mask;
- uint64_t RowIdx;
- bool CanDPP16RowShare = false;
+ if (ST.hasDPPRowShare()) {
+ uint64_t Mask;
+ uint64_t RowIdx;
+
+ // wave32 requires Mask & 0x1F == 0x10
+ // wave64 requires Mask & 0x3F == 0x30
+ uint64_t MaskCheck = (1UL << ST.getWavefrontSizeLog2()) - 1;
+ uint64_t MaskTarget = MaskCheck & 0xF0;
+
+ // DPP16 Row Share 0: Idx = Lid & Mask
+ auto RowShare0Pred = m_c_And(m_Value(Lid), m_ConstantInt(Mask));
+
+ // DPP16 Row Share (0 < Row < 15): Idx = (Lid & Mask) | RowIdx
+ auto RowSharePred =
+ m_c_Or(m_c_And(m_Value(Lid), m_ConstantInt(Mask)), m_ConstantInt(RowIdx));
+
+ // DPP16 Row Share 15: Idx = Lid | 0xF
+ auto RowShare15Pred = m_c_Or(m_Value(Lid), m_ConstantInt<0xF>());
+
+ if (match(Idx, RowShare0Pred) && isLaneID(ST, Lid) &&
+ (Mask & MaskCheck) == MaskTarget) {
+ CanDppOpt = true;
+ DppMode = AMDGPU::DPP::ROW_SHARE0;
+ } else if (match(Idx, RowSharePred) && isLaneID(ST, Lid) && RowIdx < 15 &&
+ RowIdx > 0 && (Mask & MaskCheck) == MaskTarget) {
+ CanDppOpt = true;
+ DppMode = AMDGPU::DPP::ROW_SHARE0 | RowIdx;
+ } else if (match(Idx, RowShare15Pred) && isLaneID(ST, Lid)) {
+ CanDppOpt = true;
+ DppMode = AMDGPU::DPP::ROW_SHARE0 | RowIdx;
+ }
+ }
- // wave32 requires Mask & 0x1F == 0x10
- // wave64 requires Mask & 0x3F == 0x30
- uint64_t MaskCheck = (1UL << ST.getWavefrontSizeLog2()) - 1;
- uint64_t MaskTarget = MaskCheck & 0xF0;
+ if (!CanDppOpt && ST.hasDPPRowXMask()) {
+ uint64_t Mask;
+ uint64_t XorIdx;
- // DPP16 Row Share 0: Idx = Tid & Mask
- auto RowShare0Pred = m_And(m_Value(Tid), m_ConstantInt(Mask));
+ // wave32 requires Mask & 0x1F == 0x1F
+ // wave64 requires Mask & 0x3F == 0x3F
+ uint64_t MaskCheck = (1UL << ST.getWavefrontSizeLog2()) - 1;
+ uint64_t MaskTarget = MaskCheck;
- // DPP16 Row Share (0 < Row < 15): Idx = (Tid & Mask) | RowIdx
- auto RowSharePred =
- m_Or(m_And(m_Value(Tid), m_ConstantInt(Mask)), m_ConstantInt(RowIdx));
+ // It's fine to have a mask, but also unnecessary
+ auto RowXMaskPred =
+ m_c_Xor(m_c_And(m_Value(Lid), m_ConstantInt(Mask)), m_ConstantInt(XorIdx));
- // DPP16 Row Share 15: Idx = Tid | 0xF
- auto RowShare15Pred = m_Or(m_Value(Tid), m_ConstantInt<0xF>());
+ auto RowXMaskPredNoMask = m_c_Xor(m_Value(Lid), m_ConstantInt(XorIdx));
- if (match(Idx, RowShare0Pred) && isThreadID(ST, Tid)) {
- if ((Mask & MaskCheck) != MaskTarget)
- return std::nullopt;
+ if (match(Idx, RowXMaskPred) && isLaneID(ST, Lid) &&
+ (Mask & MaskCheck) == MaskTarget && XorIdx <= 15 && XorIdx >= 0) {
+ CanDppOpt = true;
+ DppMode = AMDGPU::DPP::ROW_XMASK0 | XorIdx;
+ }
+ else if (match(Idx, RowXMaskPredNoMask) && isLaneID(ST, Lid) && XorIdx <= 15 &&
+ XorIdx >= 0) {
+ CanDppOpt = true;
+ DppMode = AMDGPU::DPP::ROW_XMASK0 | XorIdx;
+ }
+ }
- RowIdx = 0;
- CanDPP16RowShare = true;
- } else if (match(Idx, RowSharePred) && isThreadID(ST, Tid) && RowIdx < 15 &&
- RowIdx > 0) {
- if ((Mask & MaskCheck) != MaskTarget)
- return std::nullopt;
+ if (!CanDppOpt && ST.hasDPPRowMirror()) {
+ uint64_t Mask = 0xFFFF;
+
+ // wave32 requires Mask & 0x1F == 0x10
+ // wave64 requires Mask & 0x3F == 0x30
+ uint64_t MaskCheck = (1UL << ST.getWavefrontSizeLog2()) - 1;
+ uint64_t MaskTarget = MaskCheck & 0xF0;
+ uint64_t XorMaskTarget = MaskCheck;
+
+ // The actual "mirror" part can be accomplished with a SUB or an XOR,
+ // they're equivalent because the minuend is all ones
+ auto RowMirrorPred = m_c_Or(m_c_And(m_Value(Lid), m_ConstantInt(Mask)),
+ m_CombineOr(m_Sub(m_ConstantInt<0xF>(), m_c_And(m_Deferred(Lid), m_ConstantInt<0xF>())),
+ m_c_Xor(m_ConstantInt<0xF>(), m_c_And(m_Deferred(Lid), m_ConstantInt<0xF>()))));
- CanDPP16RowShare = true;
- } else if (match(Idx, RowShare15Pred) && isThreadID(ST, Tid)) {
- RowIdx = 15;
- CanDPP16RowShare = true;
+ // More optimized case, can just directly XOR
+ auto RowMirrorXorPred = m_c_Xor(m_ConstantInt<0xF>(), m_CombineOr(m_c_And(m_Value(Lid), m_ConstantInt(Mask)), m_Value(Lid)));
+
+ if (match(Idx, RowMirrorPred) && isLaneID(ST, Lid) && (Mask & MaskCheck) == MaskTarget) {
+ CanDppOpt = true;
+ DppMode = AMDGPU::DPP::ROW_MIRROR;
+ } else if (match(Idx, RowMirrorXorPred) && isLaneID(ST, Lid) && (Mask & MaskCheck) == XorMaskTarget) {
+ CanDppOpt = true;
+ DppMode = AMDGPU::DPP::ROW_MIRROR;
+ }
+ }
+
+ if (!CanDppOpt && ST.hasDPPRowHalfMirror()) {
+ uint64_t Mask = 0xFFFF;
+
+ // wave32 requires Mask & 0x1F == 0x18
+ // wave64 requires Mask & 0x3F == 0x38
+ uint64_t MaskCheck = (1UL << ST.getWavefrontSizeLog2()) - 1;
+ uint64_t MaskTarget = MaskCheck & 0xF8;
+ uint64_t XorMaskTarget = MaskCheck;
+
+ // The actual "mirror" part can be accomplished with a SUB or an XOR,
+ // they're equivalent because the minuend is all ones
+ auto RowMirrorPred = m_c_Or(m_c_And(m_Value(Lid), m_ConstantInt(Mask)),
+ m_CombineOr(m_Sub(m_ConstantInt<0x7>(), m_c_And(m_Deferred(Lid), m_ConstantInt<0x7>())),
+ m_c_Xor(m_ConstantInt<0x7>(), m_c_And(m_Deferred(Lid), m_ConstantInt<0x7>()))));
+
+ // More optimized case, can just directly XOR
+ auto RowHalfMirrorXorPred = m_c_Xor(m_ConstantInt<0x7>(), m_CombineOr(m_c_And(m_Value(Lid), m_ConstantInt(Mask)), m_Value(Lid)));
+
+ if (match(Idx, RowMirrorPred) && isLaneID(ST, Lid) && (Mask & MaskCheck) == MaskTarget) {
+ CanDppOpt = true;
+ DppMode = AMDGPU::DPP::ROW_HALF_MIRROR;
+ } else if (match(Idx, RowHalfMirrorXorPred) && isLaneID(ST, Lid) && (Mask & MaskCheck) == XorMaskTarget) {
+ CanDppOpt = true;
+ DppMode = AMDGPU::DPP::ROW_HALF_MIRROR;
+ }
}
- if (CanDPP16RowShare) {
- CallInst *UpdateDPP =
- B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, Val->getType(),
- {PoisonValue::get(Val->getType()), Val,
- B.getInt32(AMDGPU::DPP::ROW_SHARE0 | RowIdx),
- B.getInt32(0xF), B.getInt32(0xF), B.getFalse()});
- UpdateDPP->takeName(&II);
- UpdateDPP->copyMetadata(II);
- return IC.replaceInstUsesWith(II, UpdateDPP);
+ if (CanDppOpt) {
+ CallInst *UpdateDPP =
+ B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, Val->getType(),
+ {PoisonValue::get(Val->getType()), Val,
+ B.getInt32(DppMode),
+ B.getInt32(0xF), B.getInt32(0xF), B.getFalse()});
+ UpdateDPP->takeName(&II);
+ UpdateDPP->copyMetadata(II);
+ return IC.replaceInstUsesWith(II, UpdateDPP);
}
// No valid DPP detected
@@ -1843,10 +1925,11 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
return IC.eraseInstFromFunction(II);
}
case Intrinsic::amdgcn_wave_shuffle: {
- if (!ST->hasDPP())
- return std::nullopt;
+ // If the index is just the lane ID, then it's a no-op
+ if (isLaneID(*ST, II.getArgOperand(1)))
+ return IC.replaceInstUsesWith(II, II.getArgOperand(0));
- return tryWaveShuffleDPP(*ST, IC, II);
+ return tryWaveShuffleDpp(*ST, IC, II);
}
}
if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index b308e0d77305f..9638465bce7fc 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -479,6 +479,18 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
return HasDPP && (HasGFX90AInsts || getGeneration() >= GFX10);
}
+ bool hasDPPRowXMask() const {
+ return HasDPP && getGeneration() >= GFX10;
+ }
+
+ bool hasDPPRowMirror() const {
+ return HasDPP;
+ }
+
+ bool hasDPPRowHalfMirror() const {
+ return HasDPP;
+ }
+
// Has V_PK_MOV_B32 opcode
bool hasPkMovB32() const { return HasGFX90AInsts; }
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
index 4617592c70f7a..c5f6c22334e13 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
@@ -1,34 +1,67 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1100 -passes=instcombine -S < %s | FileCheck -check-prefixes=CHECK-W32 %s
-; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1100 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck -check-prefixes=CHECK-W64 %s
+; RUN: opt -mtriple=amdgcn-- -mcpu=hawaii -passes=instcombine -S < %s | FileCheck -check-prefixes=GFX7 %s
-; DPP16 Row Share optimization depends on knowing the wavefront size, this run should skip the optimization
-; RUN: opt -mtriple=amdgcn-- -mattr=+dpp -passes=instcombine -S < %s | FileCheck -check-prefixes=CHECK-NO-WAVE-SIZE %s
+; RUN: opt -mtriple=amdgcn-- -mcpu=fiji -passes=instcombine -S < %s | FileCheck -check-prefixes=GFX8 %s
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx90a -passes=instcombine -S < %s | FileCheck -check-prefixes=GFX9 %s
+
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1010 -passes=instcombine -S < %s | FileCheck -check-prefixes=GFX10-W32 %s
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1010 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck -check-prefixes=GFX10-W64 %s
+
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1100 -passes=instcombine -S < %s | FileCheck -check-prefixes=GFX11-W32 %s
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1100 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck -check-prefixes=GFX11-W64 %s
+
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1200 -passes=instcombine -S < %s | FileCheck -check-prefixes=GFX12-W32 %s
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1200 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck -check-prefixes=GFX12-W64 %s
+
+; Several optimizations depend on knowing the wavefront size, this run should skip those optimizations
+; RUN: opt -mtriple=amdgcn-- -mattr=+dpp -passes=instcombine -S < %s | FileCheck -check-prefixes=NO-INFO-CHECK %s
+
+; Using wave shuffle to select self lane is a noop
define i32 @test_wave_shuffle_self_select(i32 %val) {
-; CHECK-W32-LABEL: define i32 @test_wave_shuffle_self_select(
-; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-W32-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[TID]])
-; CHECK-W32-NEXT: ret i32 [[RES]]
+; GFX7-LABEL: define i32 @test_wave_shuffle_self_select(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX7-NEXT: ret i32 [[VAL]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_self_select(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX8-NEXT: ret i32 [[VAL]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_self_select(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX9-NEXT: ret i32 [[VAL]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_self_select(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX10-W32-NEXT: ret i32 [[VAL]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_self_select(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX10-W64-NEXT: ret i32 [[VAL]]
;
-; CHECK-W64-LABEL: define i32 @test_wave_shuffle_self_select(
-; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-W64-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W64-NEXT: [[TID1:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[TID]])
-; CHECK-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[TID1]])
-; CHECK-W64-NEXT: ret i32 [[RES]]
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_self_select(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX11-W32-NEXT: ret i32 [[VAL]]
;
-; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_self_select(
-; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NO-WAVE-SIZE-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-NO-WAVE-SIZE-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-NO-WAVE-SIZE-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[TID]])
-; CHECK-NO-WAVE-SIZE-NEXT: ret i32 [[RES]]
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_self_select(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX11-W64-NEXT: ret i32 [[VAL]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_self_select(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX12-W32-NEXT: ret i32 [[VAL]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_self_select(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX12-W64-NEXT: ret i32 [[VAL]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_self_select(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
+; NO-INFO-CHECK-NEXT: ret i32 [[VAL]]
;
%lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
- %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
- %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %tid)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %lid)
ret i32 %res
}
@@ -36,113 +69,292 @@ define i32 @test_wave_shuffle_self_select(i32 %val) {
; In fact, the or will likely be optimized out before reaching the DPP
; optimization step anyway. So this case should work with or without the or
define i32 @test_wave_shuffle_dpp_row_share_0(i32 %val) {
-; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
-; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
-; CHECK-W32-NEXT: ret i32 [[RES]]
-;
-; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
-; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
-; CHECK-W64-NEXT: ret i32 [[RES]]
-;
-; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
-; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-NO-WAVE-SIZE-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-NO-WAVE-SIZE-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-NO-WAVE-SIZE-NEXT: [[MASKED:%.*]] = and i32 [[TID]], 65520
-; CHECK-NO-WAVE-SIZE-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[MASKED]])
-; CHECK-NO-WAVE-SIZE-NEXT: ret i32 [[RES]]
-;
- %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
- %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
- %masked = and i32 %tid, 65520 ; 0xFFF0
+; GFX7-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[MASKED]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[MASKED]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[MASKED]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %masked = and i32 %lid, 65520 ; 0xFFF0
%share_0 = or i32 %masked, 0
%res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %share_0)
ret i32 %res
}
define i32 @test_wave_shuffle_dpp_row_share_0_no_or(i32 %val) {
-; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
-; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
-; CHECK-W32-NEXT: ret i32 [[RES]]
-;
-; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
-; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
-; CHECK-W64-NEXT: ret i32 [[RES]]
-;
-; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
-; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-NO-WAVE-SIZE-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-NO-WAVE-SIZE-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-NO-WAVE-SIZE-NEXT: [[MASKED:%.*]] = and i32 [[TID]], 65520
-; CHECK-NO-WAVE-SIZE-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[MASKED]])
-; CHECK-NO-WAVE-SIZE-NEXT: ret i32 [[RES]]
-;
- %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
- %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
- %masked = and i32 %tid, 65520 ; 0xFFF0
+; GFX7-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[MASKED]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[MASKED]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[MASKED]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %masked = and i32 %lid, 65520 ; 0xFFF0
%res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %masked)
ret i32 %res
}
define i32 @test_wave_shuffle_dpp_row_share_7(i32 %val) {
-; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
-; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
-; CHECK-W32-NEXT: ret i32 [[RES]]
-;
-; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
-; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
-; CHECK-W64-NEXT: ret i32 [[RES]]
-;
-; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
-; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-NO-WAVE-SIZE-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-NO-WAVE-SIZE-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-NO-WAVE-SIZE-NEXT: [[MASKED:%.*]] = and i32 [[TID]], 48
-; CHECK-NO-WAVE-SIZE-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
-; CHECK-NO-WAVE-SIZE-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
-; CHECK-NO-WAVE-SIZE-NEXT: ret i32 [[RES]]
-;
- %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
- %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
- %masked = and i32 %tid, 48 ; 0x30
+; GFX7-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 48
+; GFX7-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 48
+; GFX8-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 48
+; NO-INFO-CHECK-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %masked = and i32 %lid, 48 ; 0x30
%share_7 = or i32 %masked, 7
%res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %share_7)
ret i32 %res
}
define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(i32 %val) {
-; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
-; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W32-NEXT: [[SHARE_7:%.*]] = or i32 [[TID]], 7
-; CHECK-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
-; CHECK-W32-NEXT: ret i32 [[RES]]
-;
-; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
-; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W64-NEXT: [[TID1:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[TID]])
-; CHECK-W64-NEXT: [[SHARE_7:%.*]] = or i32 [[TID1]], 7
-; CHECK-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
-; CHECK-W64-NEXT: ret i32 [[RES]]
-;
-; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
-; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-NO-WAVE-SIZE-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-NO-WAVE-SIZE-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-NO-WAVE-SIZE-NEXT: [[SHARE_7:%.*]] = or i32 [[TID]], 7
-; CHECK-NO-WAVE-SIZE-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
-; CHECK-NO-WAVE-SIZE-NEXT: ret i32 [[RES]]
-;
- %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
- %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
- %share_7 = or i32 %tid, 7
+; GFX7-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[SHARE_7:%.*]] = or i32 [[LID]], 7
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[SHARE_7:%.*]] = or i32 [[LID]], 7
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT: [[SHARE_7:%.*]] = or i32 [[LID]], 7
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W32-NEXT: [[SHARE_7:%.*]] = or i32 [[LO]], 7
+; GFX10-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX10-W64-NEXT: [[SHARE_7:%.*]] = or i32 [[LID]], 7
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W32-NEXT: [[SHARE_7:%.*]] = or i32 [[LO]], 7
+; GFX11-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX11-W64-NEXT: [[SHARE_7:%.*]] = or i32 [[LID]], 7
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W32-NEXT: [[SHARE_7:%.*]] = or i32 [[LO]], 7
+; GFX12-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX12-W64-NEXT: [[SHARE_7:%.*]] = or i32 [[LID]], 7
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[SHARE_7:%.*]] = or i32 [[LID]], 7
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %share_7 = or i32 %lid, 7
%res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %share_7)
ret i32 %res
}
@@ -151,29 +363,79 @@ define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(i32 %val) {
; mbcnt.hi is optimized away for wave32. However, ommitting mbcnt.hi should prevent
; wave64 from optimizing to dpp.
define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(i32 %val) {
-; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
-; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
-; CHECK-W32-NEXT: ret i32 [[RES]]
-;
-; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
-; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W64-NEXT: [[MASKED:%.*]] = and i32 [[TID]], 65520
-; CHECK-W64-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
-; CHECK-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
-; CHECK-W64-NEXT: ret i32 [[RES]]
-;
-; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
-; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-NO-WAVE-SIZE-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-NO-WAVE-SIZE-NEXT: [[MASKED:%.*]] = and i32 [[TID]], 65520
-; CHECK-NO-WAVE-SIZE-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
-; CHECK-NO-WAVE-SIZE-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
-; CHECK-NO-WAVE-SIZE-NEXT: ret i32 [[RES]]
-;
- %tid = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
- %masked = and i32 %tid, 65520 ; 0xFFF0
+; GFX7-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; GFX7-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; GFX8-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; GFX9-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; GFX10-W64-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; GFX11-W64-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; GFX12-W64-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65520
+; NO-INFO-CHECK-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %masked = and i32 %lid, 65520 ; 0xFFF0
%share_7 = or i32 %masked, 7
%res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %share_7)
ret i32 %res
@@ -183,68 +445,1264 @@ define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(i32 %val) {
; has 4 rows. This test has a mask that should only be valid for wave32 to be
; optimized to dpp.
define i32 @test_wave_shuffle_dpp_row_share_w32_mask(i32 %val) {
-; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
-; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
-; CHECK-W32-NEXT: ret i32 [[RES]]
-;
-; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
-; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W64-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-W64-NEXT: [[MASKED:%.*]] = and i32 [[TID]], 16
-; CHECK-W64-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
-; CHECK-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
-; CHECK-W64-NEXT: ret i32 [[RES]]
-;
-; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
-; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-NO-WAVE-SIZE-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-NO-WAVE-SIZE-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-NO-WAVE-SIZE-NEXT: [[MASKED:%.*]] = and i32 [[TID]], 16
-; CHECK-NO-WAVE-SIZE-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
-; CHECK-NO-WAVE-SIZE-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
-; CHECK-NO-WAVE-SIZE-NEXT: ret i32 [[RES]]
-;
- %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
- %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
- %masked = and i32 %tid, 16 ; 0x10
+; GFX7-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 16
+; GFX7-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 16
+; GFX8-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 16
+; GFX9-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX10-W64-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 16
+; GFX10-W64-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX11-W64-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 16
+; GFX11-W64-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX12-W64-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 16
+; GFX12-W64-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 16
+; NO-INFO-CHECK-NEXT: [[SHARE_7:%.*]] = or disjoint i32 [[MASKED]], 7
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %masked = and i32 %lid, 16 ; 0x10
%share_7 = or i32 %masked, 7
%res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %share_7)
ret i32 %res
}
define i32 @test_wave_shuffle_not_quite_row_share(i32 %val) {
-; CHECK-W32-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
-; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W32-NEXT: [[MASKED:%.*]] = and i32 [[TID]], 65280
-; CHECK-W32-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
-; CHECK-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
-; CHECK-W32-NEXT: ret i32 [[RES]]
-;
-; CHECK-W64-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
-; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W64-NEXT: [[TID1:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[TID]])
-; CHECK-W64-NEXT: [[MASKED:%.*]] = and i32 [[TID1]], 65280
-; CHECK-W64-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
-; CHECK-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
-; CHECK-W64-NEXT: ret i32 [[RES]]
-;
-; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
-; CHECK-NO-WAVE-SIZE-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-NO-WAVE-SIZE-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-NO-WAVE-SIZE-NEXT: [[TID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-NO-WAVE-SIZE-NEXT: [[MASKED:%.*]] = and i32 [[TID]], 65280
-; CHECK-NO-WAVE-SIZE-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
-; CHECK-NO-WAVE-SIZE-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
-; CHECK-NO-WAVE-SIZE-NEXT: ret i32 [[RES]]
-;
- %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
- %tid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
- %masked = and i32 %tid, 65280 ; 0xFF00
+; GFX7-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65280
+; GFX7-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65280
+; GFX8-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65280
+; GFX9-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W32-NEXT: [[MASKED:%.*]] = and i32 [[LO]], 65280
+; GFX10-W32-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
+; GFX10-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX10-W64-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65280
+; GFX10-W64-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W32-NEXT: [[MASKED:%.*]] = and i32 [[LO]], 65280
+; GFX11-W32-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
+; GFX11-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX11-W64-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65280
+; GFX11-W64-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W32-NEXT: [[MASKED:%.*]] = and i32 [[LO]], 65280
+; GFX12-W32-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
+; GFX12-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX12-W64-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65280
+; GFX12-W64-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_not_quite_row_share(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[MASKED:%.*]] = and i32 [[LID]], 65280
+; NO-INFO-CHECK-NEXT: [[OR_RES:%.*]] = or disjoint i32 [[MASKED]], 55
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[OR_RES]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %masked = and i32 %lid, 65280 ; 0xFF00
%or_res = or i32 %masked, 55 ; 0x37
%res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %or_res)
ret i32 %res
}
+
+define i32 @test_wave_shuffle_row_xmask(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_xmask(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_xmask(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX8-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_xmask(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_xmask(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_xmask(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_xmask(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_xmask(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_xmask(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_xmask(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_xmask(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %row = and i32 %lid, 65520 ; 0xFFF0
+ %row_idx = and i32 %lid, 15 ; 0xF
+ %xor_res = xor i32 %row_idx, 5 ; 0x5
+ %idx = or i32 %row, %xor_res
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+; While it is okay to do a mask-off and or to compute the index, since
+; we know the lane ID is 32(64) or less, it's not necessary. So the
+; optimization should work with or without the masking
+define i32 @test_wave_shuffle_row_xmask_simple(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_xmask_simple(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 5
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_xmask_simple(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 5
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_xmask_simple(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 5
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_simple(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_simple(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_simple(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_simple(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_simple(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_simple(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_xmask_simple(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 5
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %idx = xor i32 %lid, 5 ; 0x5
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+; XMask with 0 is effectively a self-select, should be noop
+define i32 @test_wave_shuffle_row_xmask_0(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_xmask_0(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: ret i32 [[VAL]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_xmask_0(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: ret i32 [[VAL]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_xmask_0(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: ret i32 [[VAL]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_0(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: ret i32 [[VAL]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_0(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: ret i32 [[VAL]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_0(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: ret i32 [[VAL]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_0(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: ret i32 [[VAL]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_0(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: ret i32 [[VAL]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_0(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: ret i32 [[VAL]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_xmask_0(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: ret i32 [[VAL]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %idx = xor i32 %lid, 0 ; 0x0
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+; This case of Row XMask is equivalent to Row Mirror. On architectures
+; that have Row Mirror but not Row XMask, this case should still
+; optimize to Row Mirror
+define i32 @test_wave_shuffle_row_xmask_15(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_xmask_15(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 15
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_xmask_15(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 320, i32 15, i32 15, i1 false)
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_xmask_15(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 320, i32 15, i32 15, i1 false)
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_15(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_15(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_15(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_15(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_15(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_15(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_xmask_15(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 15
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %idx = xor i32 %lid, 15 ; 0xF
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+; This case of Row XMask is equivalent to Row Half Mirror. On architectures
+; that have Row Half Mirror but not Row XMask, this case should still
+; optimize to Row Half Mirror
+define i32 @test_wave_shuffle_row_xmask_7(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_xmask_7(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 7
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_xmask_7(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 321, i32 15, i32 15, i1 false)
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_xmask_7(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 321, i32 15, i32 15, i1 false)
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_7(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_7(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_7(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_7(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_7(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_7(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_xmask_7(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 7
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %idx = xor i32 %lid, 7 ; 0x7
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+; While masks are not necessary for this optimization, if there is a mask then
+; it does have to be of proper form. The mask requirements for wave32 and wave64
+; are slightly different since wave64 has 4 rows. This test has a mask that should
+; only be valid for wave32 to be optimized to dpp.
+define i32 @test_wave_shuffle_row_xmask_w32_mask(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_xmask_w32_mask(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_xmask_w32_mask(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX8-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_xmask_w32_mask(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_w32_mask(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_w32_mask(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX10-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX10-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_w32_mask(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_w32_mask(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX11-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_w32_mask(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 357, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_w32_mask(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX12-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX12-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_xmask_w32_mask(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 5
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %row = and i32 %lid, 16 ; 0x10
+ %row_idx = and i32 %lid, 15 ; 0xF
+ %xor_res = xor i32 %row_idx, 5 ; 0x5
+ %idx = or i32 %row, %xor_res
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+; Doing both mbcnt.lo and mbcnt.hi works for both wave32 and wave64 because the
+; mbcnt.hi is optimized away for wave32. However, ommitting mbcnt.hi should prevent
+; wave64 from optimizing to dpp.
+define i32 @test_wave_shuffle_row_xmask_lo_only(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_xmask_lo_only(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 15
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_xmask_lo_only(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 15
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_xmask_lo_only(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 15
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_lo_only(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_lo_only(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 15
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_lo_only(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_lo_only(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 15
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_xmask_lo_only(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_xmask_lo_only(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 15
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_xmask_lo_only(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[LID]], 15
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lid, 15 ; 0xF
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+; Similar to before, Row Mirror is equivalent to Row XMask 15. So, architectures
+; with Row XMask should optimize to that instead
+define i32 @test_wave_shuffle_row_mirror(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_mirror(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_mirror(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 320, i32 15, i32 15, i1 false)
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_mirror(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 320, i32 15, i32 15, i1 false)
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_mirror(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_mirror(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_mirror(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_mirror(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_mirror(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_mirror(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_mirror(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %row = and i32 %lid, 65520 ; 0xFFF0
+ %row_idx = and i32 %lid, 15 ; 0xF
+ %sub_res = sub i32 15, %row_idx ; 0xF
+ %idx = or i32 %row, %sub_res
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+define i32 @test_wave_shuffle_row_mirror_w32_mask(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_mirror_w32_mask(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_mirror_w32_mask(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX8-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_mirror_w32_mask(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_mirror_w32_mask(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_mirror_w32_mask(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX10-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX10-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_mirror_w32_mask(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_mirror_w32_mask(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX11-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_mirror_w32_mask(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_mirror_w32_mask(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX12-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX12-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_mirror_w32_mask(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %row = and i32 %lid, 16 ; 0x10
+ %row_idx = and i32 %lid, 15 ; 0xF
+ %sub_res = sub i32 15, %row_idx ; 0xF
+ %idx = or i32 %row, %sub_res
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+define i32 @test_wave_shuffle_row_mirror_lo_only(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_mirror_lo_only(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_mirror_lo_only(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX8-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_mirror_lo_only(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_mirror_lo_only(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_mirror_lo_only(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX10-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_mirror_lo_only(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_mirror_lo_only(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_mirror_lo_only(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 367, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_mirror_lo_only(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX12-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_mirror_lo_only(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 15
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %row = and i32 %lid, 65520 ; 0xFFF0
+ %row_idx = and i32 %lid, 15 ; 0xF
+ %sub_res = sub i32 15, %row_idx ; 0xF
+ %idx = or i32 %row, %sub_res
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+; Similar to before, Row Half Mirror is equivalent to Row XMask 7. So, architectures
+; with Row XMask should optimize to that instead
+define i32 @test_wave_shuffle_row_half_mirror(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_half_mirror(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_half_mirror(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 321, i32 15, i32 15, i1 false)
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_half_mirror(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 321, i32 15, i32 15, i1 false)
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_half_mirror(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_half_mirror(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_half_mirror(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_half_mirror(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_half_mirror(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_half_mirror(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_half_mirror(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %row = and i32 %lid, 65528 ; 0xFFF8
+ %row_idx = and i32 %lid, 7 ; 0x7
+ %sub_res = sub i32 7, %row_idx ; 0x7
+ %idx = or i32 %row, %sub_res
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+define i32 @test_wave_shuffle_row_half_mirror_w32_mask(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_half_mirror_w32_mask(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_half_mirror_w32_mask(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX8-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_half_mirror_w32_mask(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_half_mirror_w32_mask(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_half_mirror_w32_mask(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX10-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX10-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_half_mirror_w32_mask(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_half_mirror_w32_mask(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX11-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_half_mirror_w32_mask(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_half_mirror_w32_mask(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX12-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; GFX12-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_half_mirror_w32_mask(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 31
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %row = and i32 %lid, 24 ; 0x18
+ %row_idx = and i32 %lid, 7 ; 0x7
+ %sub_res = sub i32 7, %row_idx ; 0x7
+ %idx = or i32 %row, %sub_res
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+define i32 @test_wave_shuffle_row_half_mirror_lo_only(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_row_half_mirror_lo_only(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_row_half_mirror_lo_only(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX8-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_row_half_mirror_lo_only(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_row_half_mirror_lo_only(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_row_half_mirror_lo_only(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX10-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_row_half_mirror_lo_only(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_row_half_mirror_lo_only(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_row_half_mirror_lo_only(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 359, i32 15, i32 15, i1 false)
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_row_half_mirror_lo_only(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; GFX12-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_row_half_mirror_lo_only(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65535
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %row = and i32 %lid, 65528 ; 0xFFF8
+ %row_idx = and i32 %lid, 7 ; 0x7
+ %sub_res = sub i32 7, %row_idx ; 0x7
+ %idx = or i32 %row, %sub_res
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
+
+define i32 @test_wave_shuffle_not_quite_row_half_mirror(i32 %val) {
+; GFX7-LABEL: define i32 @test_wave_shuffle_not_quite_row_half_mirror(
+; GFX7-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX7-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX7-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65527
+; GFX7-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX7-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX7-NEXT: ret i32 [[RES]]
+;
+; GFX8-LABEL: define i32 @test_wave_shuffle_not_quite_row_half_mirror(
+; GFX8-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX8-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX8-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65527
+; GFX8-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX8-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX8-NEXT: ret i32 [[RES]]
+;
+; GFX9-LABEL: define i32 @test_wave_shuffle_not_quite_row_half_mirror(
+; GFX9-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65527
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX9-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RES]]
+;
+; GFX10-W32-LABEL: define i32 @test_wave_shuffle_not_quite_row_half_mirror(
+; GFX10-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W32-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W32-NEXT: [[TMP1:%.*]] = and i32 [[LO]], 65527
+; GFX10-W32-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX10-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX10-W32-NEXT: ret i32 [[RES]]
+;
+; GFX10-W64-LABEL: define i32 @test_wave_shuffle_not_quite_row_half_mirror(
+; GFX10-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX10-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX10-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX10-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65527
+; GFX10-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX10-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX10-W64-NEXT: ret i32 [[RES]]
+;
+; GFX11-W32-LABEL: define i32 @test_wave_shuffle_not_quite_row_half_mirror(
+; GFX11-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W32-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W32-NEXT: [[TMP1:%.*]] = and i32 [[LO]], 65527
+; GFX11-W32-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX11-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX11-W32-NEXT: ret i32 [[RES]]
+;
+; GFX11-W64-LABEL: define i32 @test_wave_shuffle_not_quite_row_half_mirror(
+; GFX11-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX11-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX11-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65527
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX11-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RES]]
+;
+; GFX12-W32-LABEL: define i32 @test_wave_shuffle_not_quite_row_half_mirror(
+; GFX12-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W32-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W32-NEXT: [[TMP1:%.*]] = and i32 [[LO]], 65527
+; GFX12-W32-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX12-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX12-W32-NEXT: ret i32 [[RES]]
+;
+; GFX12-W64-LABEL: define i32 @test_wave_shuffle_not_quite_row_half_mirror(
+; GFX12-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; GFX12-W64-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX12-W64-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX12-W64-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65527
+; GFX12-W64-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; GFX12-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; GFX12-W64-NEXT: ret i32 [[RES]]
+;
+; NO-INFO-CHECK-LABEL: define i32 @test_wave_shuffle_not_quite_row_half_mirror(
+; NO-INFO-CHECK-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
+; NO-INFO-CHECK-NEXT: [[LO:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; NO-INFO-CHECK-NEXT: [[LID:%.*]] = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; NO-INFO-CHECK-NEXT: [[TMP1:%.*]] = and i32 [[LID]], 65527
+; NO-INFO-CHECK-NEXT: [[IDX:%.*]] = xor i32 [[TMP1]], 7
+; NO-INFO-CHECK-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[IDX]])
+; NO-INFO-CHECK-NEXT: ret i32 [[RES]]
+;
+ %lo = tail call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lid = tail call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %row = and i32 %lid, 65520 ; 0xFFF0
+ %row_idx = and i32 %lid, 7 ; 0x7
+ %sub_res = sub i32 7, %row_idx ; 0x7
+ %idx = or i32 %row, %sub_res
+ %res = tail call i32 @llvm.amdgcn.wave.shuffle(i32 %val, i32 %idx)
+ ret i32 %res
+}
More information about the llvm-commits
mailing list