[llvm] [AMDGPU][InstCombine] Optimize constant shuffle patterns (PR #192246)
Aleksandar Spasojevic via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 15 07:42:55 PDT 2026
https://github.com/aleksandar-amd updated https://github.com/llvm/llvm-project/pull/192246
>From 93f9ec2c06d8539e6f90a2bc1887db5cf4bafe18 Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Tue, 14 Apr 2026 17:07:20 +0200
Subject: [PATCH 1/2] [AMDGPU][InstCombine] Match constant shuffle patterns to
specialized forms
Detect llvm.amdgcn.wave.shuffle intrinsics where the lane index is a
constant function of the lane ID and replace them with hardware-specific
intrinsics. This avoids the expensive ds_bpermute fallback, which
requires address computation and LDS memory access, by using
register-to-register lane permutation instructions that do not add LDS
pressure.
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 334 +++++++++++++++++-
.../AMDGPU/llvm.amdgcn.wave.shuffle.ll | 16 +-
.../AMDGPU/wave-shuffle-patterns.ll | 289 +++++++++++++++
3 files changed, 621 insertions(+), 18 deletions(-)
create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 463d63a88f690..95310e7d05cca 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -17,7 +17,9 @@
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetTransformInfo.h"
#include "GCNSubtarget.h"
+#include "SIDefines.h"
#include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
@@ -718,6 +720,324 @@ GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
return nullptr;
}
+// Match llvm.amdgcn.mbcnt.lo / mbcnt.hi patterns that compute the lane ID.
+// WaveSize is needed because standalone mbcnt.lo returns lane % 32, which only
+// equals the lane ID on wave32. On wave64, the full mbcnt.hi(-1, mbcnt.lo(-1, 0))
+// chain is required.
+static bool matchLaneID(const Value *V, unsigned WaveSize) {
+ const auto *II = dyn_cast<IntrinsicInst>(V);
+ if (!II)
+ return false;
+
+ if (II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_hi) {
+ const auto *Mask = dyn_cast<ConstantInt>(II->getArgOperand(0));
+ if (Mask && Mask->getSExtValue() == -1)
+ return matchLaneID(II->getArgOperand(1), /*WaveSize=*/32);
+ return false;
+ }
+
+ if (II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo) {
+ if (WaveSize > 32)
+ return false;
+ const auto *Mask = dyn_cast<ConstantInt>(II->getArgOperand(0));
+ const auto *Base = dyn_cast<ConstantInt>(II->getArgOperand(1));
+ return Mask && Mask->getSExtValue() == -1 && Base && Base->isZero();
+ }
+
+ return false;
+}
+
+static std::optional<unsigned> evalLaneExpr(const Value *V, unsigned Lane,
+ unsigned WaveSize,
+ const DataLayout &DL,
+ unsigned Depth = 0) {
+ if (Depth > 16)
+ return std::nullopt;
+
+ if (matchLaneID(V, WaveSize))
+ return Lane;
+
+ if (const auto *CI = dyn_cast<ConstantInt>(V))
+ return CI->getZExtValue();
+
+ const auto *BO = dyn_cast<BinaryOperator>(V);
+ if (!BO)
+ return std::nullopt;
+
+ auto LHS = evalLaneExpr(BO->getOperand(0), Lane, WaveSize, DL, Depth + 1);
+ auto RHS = evalLaneExpr(BO->getOperand(1), Lane, WaveSize, DL, Depth + 1);
+ if (!LHS || !RHS)
+ return std::nullopt;
+
+ Type *Ty = BO->getType();
+ Constant *LC = ConstantInt::get(Ty, *LHS);
+ Constant *RC = ConstantInt::get(Ty, *RHS);
+ Constant *Result = ConstantFoldBinaryOpOperands(BO->getOpcode(), LC, RC, DL);
+ auto *CI = dyn_cast_or_null<ConstantInt>(Result);
+ if (!CI)
+ return std::nullopt;
+ return CI->getZExtValue();
+}
+
+static bool tryBuildShuffleMap(const Value *Index, unsigned WaveSize,
+ SmallVectorImpl<uint8_t> &Ids,
+ const DataLayout &DL) {
+ Ids.resize(WaveSize);
+ for (unsigned Lane = 0; Lane < WaveSize; ++Lane) {
+ auto Val = evalLaneExpr(Index, Lane, WaveSize, DL);
+ if (!Val || *Val >= WaveSize)
+ return false;
+ Ids[Lane] = static_cast<uint8_t>(*Val);
+ }
+ return true;
+}
+
+static bool isHalfRowPattern(ArrayRef<uint8_t> Ids) {
+ for (unsigned i = 0; i < 8; ++i)
+ if (Ids[i] >= 8)
+ return false;
+ for (unsigned i = 8, E = Ids.size(); i < E; ++i)
+ if (Ids[i] != Ids[i % 8] + (i & ~7u))
+ return false;
+ return true;
+}
+
+static bool isFullRowPattern(ArrayRef<uint8_t> Ids) {
+ for (unsigned i = 0; i < 16; ++i)
+ if (Ids[i] >= 16)
+ return false;
+ for (unsigned i = 16, E = Ids.size(); i < E; ++i)
+ if (Ids[i] != Ids[i % 16] + (i & ~15u))
+ return false;
+ return true;
+}
+
+static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
+ for (unsigned i = 0; i < 4; ++i)
+ if (Ids[i] >= 4)
+ return std::nullopt;
+ for (unsigned i = 4, E = Ids.size(); i < E; ++i)
+ if (Ids[i] != Ids[i % 4] + (i & ~3u))
+ return std::nullopt;
+ return (Ids[3] << 6) | (Ids[2] << 4) | (Ids[1] << 2) | Ids[0];
+}
+
+static bool matchHalfRowMirrorPattern(ArrayRef<uint8_t> Ids) {
+ if (!isHalfRowPattern(Ids))
+ return false;
+ for (unsigned j = 0; j < 8; ++j)
+ if (Ids[j] != 7 - j)
+ return false;
+ return true;
+}
+
+static bool matchFullRowMirrorPattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return false;
+ for (unsigned j = 0; j < 16; ++j)
+ if (Ids[j] != 15 - j)
+ return false;
+ return true;
+}
+
+static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return std::nullopt;
+ if (Ids[0] == 0 || Ids[15] == 15)
+ return std::nullopt;
+ for (unsigned j = 1; j < 16; ++j) {
+ int Diff = static_cast<int>(Ids[j]) - static_cast<int>(Ids[j - 1]);
+ if (Diff != 1 && Diff != -15)
+ return std::nullopt;
+ }
+ return 16u - Ids[0];
+}
+
+static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return std::nullopt;
+ for (unsigned j = 1; j < 16; ++j)
+ if (Ids[j] != Ids[0])
+ return std::nullopt;
+ return static_cast<unsigned>(Ids[0]);
+}
+
+static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return std::nullopt;
+ unsigned Mask = Ids[0];
+ if (Mask == 0)
+ return std::nullopt;
+ for (unsigned j = 0; j < 16; ++j)
+ if (Ids[j] != (Mask ^ j))
+ return std::nullopt;
+ return Mask;
+}
+
+static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
+ if (!isHalfRowPattern(Ids))
+ return std::nullopt;
+ unsigned Selector = 0;
+ for (unsigned j = 0; j < 8; ++j)
+ Selector |= static_cast<unsigned>(Ids[j] & 0x7) << (j * 3);
+ return Selector;
+}
+
+static std::pair<uint32_t, uint32_t>
+computePermlane16Masks(ArrayRef<uint8_t> Ids) {
+ uint32_t Lo = 0, Hi = 0;
+ for (unsigned j = 0; j < 8; ++j)
+ Lo |= static_cast<uint32_t>(Ids[j] & 0xF) << (j * 4);
+ for (unsigned j = 8; j < 16; ++j)
+ Hi |= static_cast<uint32_t>(Ids[j] & 0xF) << ((j - 8) * 4);
+ return {Lo, Hi};
+}
+
+static std::optional<unsigned> matchHalfRowSharePattern(ArrayRef<uint8_t> Ids) {
+ if (!isHalfRowPattern(Ids))
+ return std::nullopt;
+ for (unsigned j = 1; j < 8; ++j)
+ if (Ids[j] != Ids[0])
+ return std::nullopt;
+ // Bitmask-mode: AND=0x18 (keep group-select bits), OR=ids[0], XOR=0
+ return (static_cast<unsigned>(Ids[0]) << AMDGPU::Swizzle::BITMASK_OR_SHIFT) |
+ 0x18u;
+}
+
+static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
+ if (Ids.size() != 64)
+ return false;
+ for (unsigned j = 0; j < 64; ++j)
+ if (Ids[j] != (j ^ 32))
+ return false;
+ return true;
+}
+
+static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl,
+ Module *M) {
+ Type *Ty = Val->getType();
+ Function *F =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_update_dpp, {Ty});
+ return B.CreateCall(F, {PoisonValue::get(Ty), Val, B.getInt32(Ctrl),
+ B.getInt32(0xF), B.getInt32(0xF), B.getTrue()});
+}
+
+static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector,
+ Module *M) {
+ Type *Ty = Val->getType();
+ Function *F =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_mov_dpp8, {Ty});
+ return B.CreateCall(F, {Val, B.getInt32(Selector)});
+}
+
+static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
+ uint32_t Hi, Module *M) {
+ Type *Ty = Val->getType();
+ Function *F =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_permlane16, {Ty});
+ return B.CreateCall(F, {PoisonValue::get(Ty), Val, B.getInt32(Lo),
+ B.getInt32(Hi), B.getFalse(), B.getFalse()});
+}
+
+static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
+ Module *M) {
+ Type *OrigTy = Val->getType();
+ Value *Src = Val;
+ if (!OrigTy->isIntegerTy(32))
+ Src = B.CreateBitCast(Src, B.getInt32Ty());
+ Function *F =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_ds_swizzle);
+ Value *Result = B.CreateCall(F, {Src, B.getInt32(Offset)});
+ if (!OrigTy->isIntegerTy(32))
+ Result = B.CreateBitCast(Result, OrigTy);
+ return Result;
+}
+
+static Value *createPermlane64(IRBuilderBase &B, Value *Val, Module *M) {
+ Type *Ty = Val->getType();
+ Function *F =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_permlane64, {Ty});
+ return B.CreateCall(F, {Val});
+}
+
+/// Try to fold a wave_shuffle whose lane index is a constant function of the
+/// lane ID into a hardware-specific lane permutation intrinsic.
+static std::optional<Instruction *>
+tryOptimizeWaveShuffle(InstCombiner &IC, IntrinsicInst &II,
+ const GCNSubtarget &ST) {
+ if (II.getType()->getPrimitiveSizeInBits() != 32)
+ return std::nullopt;
+
+ if (!ST.isWaveSizeKnown())
+ return std::nullopt;
+
+ unsigned WaveSize = ST.getWavefrontSize();
+ Value *Src = II.getArgOperand(0);
+ Value *Index = II.getArgOperand(1);
+
+ SmallVector<uint8_t, 64> Ids;
+ if (!tryBuildShuffleMap(Index, WaveSize, Ids, IC.getDataLayout()))
+ return std::nullopt;
+
+ IRBuilderBase &B = IC.Builder;
+ Module *M = II.getModule();
+ Value *Result = nullptr;
+
+ // Try patterns in priority order: most specific first.
+ if (auto QP = matchQuadPermPattern(Ids)) {
+ if (ST.hasDPP())
+ Result = createUpdateDpp(B, Src, *QP, M);
+ else
+ Result = createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, M);
+ }
+
+ if (!Result && ST.hasDPP() && matchHalfRowMirrorPattern(Ids))
+ Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_HALF_MIRROR, M);
+
+ if (!Result && ST.hasDPP() && matchFullRowMirrorPattern(Ids))
+ Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_MIRROR, M);
+
+ if (!Result && ST.hasDPP()) {
+ if (auto Amt = matchRowRotatePattern(Ids))
+ Result =
+ createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1, M);
+ }
+
+ if (!Result && ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
+ if (auto Lane = matchRowSharePattern(Ids))
+ Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane, M);
+ }
+
+ if (!Result && ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
+ if (auto Mask = matchRowXMaskPattern(Ids))
+ Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask, M);
+ }
+
+ if (!Result && ST.hasDPP8()) {
+ if (auto Sel = matchHalfRowPermPattern(Ids))
+ Result = createMovDpp8(B, Src, *Sel, M);
+ }
+
+ if (!Result && ST.hasPermLaneX16() && isFullRowPattern(Ids)) {
+ auto [Lo, Hi] = computePermlane16Masks(Ids);
+ Result = createPermlane16(B, Src, Lo, Hi, M);
+ }
+
+ // DS_SWIZZLE bitmask-mode fallback for targets without DPP8/permlane16.
+ if (!Result) {
+ if (auto Offset = matchHalfRowSharePattern(Ids))
+ Result = createDsSwizzle(B, Src, *Offset, M);
+ }
+
+ if (!Result && ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
+ Result = createPermlane64(B, Src, M);
+
+ if (!Result)
+ return std::nullopt;
+
+ return IC.replaceInstUsesWith(II, Result);
+}
+
std::optional<Instruction *>
GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Intrinsic::ID IID = II.getIntrinsicID();
@@ -1558,6 +1878,14 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
return IC.replaceOperand(II, 0, PoisonValue::get(VDstIn->getType()));
}
+ case Intrinsic::amdgcn_wave_shuffle: {
+ if (ST->hasDPP())
+ if (auto R = tryWaveShuffleDPP(*ST, IC, II))
+ return R;
+ if (auto R = tryOptimizeWaveShuffle(IC, II, *ST))
+ return R;
+ break;
+ }
case Intrinsic::amdgcn_permlane64:
case Intrinsic::amdgcn_readfirstlane:
case Intrinsic::amdgcn_readlane:
@@ -1896,12 +2224,6 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
NewII->takeName(&II);
return IC.replaceInstUsesWith(II, NewII);
}
- case Intrinsic::amdgcn_wave_shuffle: {
- if (!ST->hasDPP())
- return std::nullopt;
-
- return tryWaveShuffleDPP(*ST, IC, II);
- }
}
if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
AMDGPU::getImageDimIntrinsicInfo(II.getIntrinsicID())) {
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
index 87fc259bdebff..ed415313b941c 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
@@ -8,15 +8,12 @@
define i32 @test_wave_shuffle_self_select(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_self_select(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-W32-NEXT: [[TID:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[TID]])
+; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 228, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT: ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_self_select(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-W64-NEXT: [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W64-NEXT: [[TID1:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[TID1]])
+; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 228, i32 15, i32 15, i1 true)
; CHECK-W64-NEXT: ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_self_select(
@@ -119,17 +116,12 @@ define i32 @test_wave_shuffle_dpp_row_share_7(i32 %val) {
define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[TID:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W32-NEXT: [[SHARE_7:%.*]] = or i32 [[TID]], 7
-; CHECK-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL]], i32 16777215)
; CHECK-W32-NEXT: ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W64-NEXT: [[TID1:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-W64-NEXT: [[SHARE_7:%.*]] = or i32 [[TID1]], 7
-; CHECK-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL]], i32 16777215)
; CHECK-W64-NEXT: ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
new file mode 100644
index 0000000000000..fc42b90f72155
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
@@ -0,0 +1,289 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11-W64
+
+declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
+declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
+declare i32 @llvm.amdgcn.wave.shuffle.i32(i32, i32)
+declare float @llvm.amdgcn.wave.shuffle.f32(float, i32)
+
+; Quad perm: lane ^ 1 swaps adjacent pairs.
+define i32 @test_quad_perm_xor1_w32(i32 %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Quad perm: lane ^ 3 reverses within each quad.
+define i32 @test_quad_perm_xor3_w32(i32 %val) {
+; GFX11-LABEL: @test_quad_perm_xor3_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 27, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor3_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 3
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 3
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Half-row mirror: reverse within every 8-lane group.
+define i32 @test_half_row_mirror_w32(i32 %val) {
+; GFX11-LABEL: @test_half_row_mirror_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 321, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_half_row_mirror_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 7
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low3 = and i32 %lane, 7
+ %rev = sub i32 7, %low3
+ %high = and i32 %lane, -8
+ %idx = or i32 %high, %rev
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Full-row mirror: reverse within every 16-lane row.
+define i32 @test_full_row_mirror_w32(i32 %val) {
+; GFX11-LABEL: @test_full_row_mirror_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 320, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_full_row_mirror_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 15
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low4 = and i32 %lane, 15
+ %rev = sub i32 15, %low4
+ %high = and i32 %lane, -16
+ %idx = or i32 %high, %rev
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Row rotate right by 3 within each 16-lane row.
+define i32 @test_row_rotate_right3_w32(i32 %val) {
+; GFX11-LABEL: @test_row_rotate_right3_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 301, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_row_rotate_right3_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
+; GFX11-W64-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 15
+; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low4 = and i32 %lane, 15
+ %rot = add i32 %low4, 3
+ %wrapped = and i32 %rot, 15
+ %high = and i32 %lane, -16
+ %idx = or i32 %high, %wrapped
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Row share: broadcast lane 3 within each 16-lane row.
+define i32 @test_row_share3_w32(i32 %val) {
+; GFX11-LABEL: @test_row_share3_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 339, i32 15, i32 15, i1 false)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_row_share3_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %high = and i32 %lane, -16
+ %idx = or i32 %high, 3
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Row xmask: xor each lane with mask 5 within each 16-lane row.
+define i32 @test_row_xmask5_w32(i32 %val) {
+; GFX11-LABEL: @test_row_xmask5_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_row_xmask5_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 5
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low4 = and i32 %lane, 15
+ %xored = xor i32 %low4, 5
+ %high = and i32 %lane, -16
+ %idx = or i32 %high, %xored
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Half-wave swap (wave64 only): lane ^ 32.
+define i32 @test_half_wave_swap_w64(i32 %val) {
+; GFX11-LABEL: @test_half_wave_swap_w64(
+; GFX11-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LO]], 32
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_half_wave_swap_w64(
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane64.i32(i32 [[VAL:%.*]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %idx = xor i32 %lane, 32
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; DPP8: rotate left by 1 within each 8-lane group.
+define i32 @test_dpp8_rotate1_w32(i32 %val) {
+; GFX11-LABEL: @test_dpp8_rotate1_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL:%.*]], i32 2054353)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_dpp8_rotate1_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 7
+; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 56
+; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low3 = and i32 %lane, 7
+ %rot = add i32 %low3, 1
+ %wrapped = and i32 %rot, 7
+ %high = and i32 %lane, -8
+ %idx = or i32 %high, %wrapped
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Permlane16: (lane * 3) & 15, a bijection on Z/16Z.
+define i32 @test_permlane16_mul3_w32(i32 %val) {
+; GFX11-LABEL: @test_permlane16_mul3_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane16.i32(i32 poison, i32 [[VAL:%.*]], i32 1392285232, i32 -629924168, i1 false, i1 false)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_permlane16_mul3_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
+; GFX11-W64-NEXT: [[PERM:%.*]] = and i32 [[MUL3]], 15
+; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low4 = and i32 %lane, 15
+ %mul3 = mul i32 %low4, 3
+ %perm = and i32 %mul3, 15
+ %high = and i32 %lane, -16
+ %idx = or i32 %high, %perm
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Float type should work the same as i32.
+define float @test_quad_perm_xor1_float_w32(float %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_float_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float poison, float [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret float [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_float_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret float [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %result = call float @llvm.amdgcn.wave.shuffle.f32(float %val, i32 %idx)
+ ret float %result
+}
+
+; Negative: non-constant index, should not be optimized.
+define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
+; GFX11-LABEL: @test_nonconstant_shuffle_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_nonconstant_shuffle_w32(
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Identity shuffle: lane_id itself.
+define i32 @test_identity_shuffle_w32(i32 %val) {
+; GFX11-LABEL: @test_identity_shuffle_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 228, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_identity_shuffle_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %lane)
+ ret i32 %result
+}
+
+; Negative: index out of range (lane + 32 >= 32 for wave32).
+define i32 @test_out_of_range_w32(i32 %val) {
+; GFX11-LABEL: @test_out_of_range_w32(
+; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_out_of_range_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = add i32 %lane, 32
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
>From 2c0e0e802598f5283fcbf2221179d9f09c90c68c Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Wed, 15 Apr 2026 16:40:25 +0200
Subject: [PATCH 2/2] Resolved comments
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 117 ++++++------------
1 file changed, 41 insertions(+), 76 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 95310e7d05cca..8ea24e09a6958 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -720,52 +720,25 @@ GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
return nullptr;
}
-// Match llvm.amdgcn.mbcnt.lo / mbcnt.hi patterns that compute the lane ID.
-// WaveSize is needed because standalone mbcnt.lo returns lane % 32, which only
-// equals the lane ID on wave32. On wave64, the full mbcnt.hi(-1, mbcnt.lo(-1, 0))
-// chain is required.
-static bool matchLaneID(const Value *V, unsigned WaveSize) {
- const auto *II = dyn_cast<IntrinsicInst>(V);
- if (!II)
- return false;
-
- if (II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_hi) {
- const auto *Mask = dyn_cast<ConstantInt>(II->getArgOperand(0));
- if (Mask && Mask->getSExtValue() == -1)
- return matchLaneID(II->getArgOperand(1), /*WaveSize=*/32);
- return false;
- }
-
- if (II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo) {
- if (WaveSize > 32)
- return false;
- const auto *Mask = dyn_cast<ConstantInt>(II->getArgOperand(0));
- const auto *Base = dyn_cast<ConstantInt>(II->getArgOperand(1));
- return Mask && Mask->getSExtValue() == -1 && Base && Base->isZero();
- }
-
- return false;
-}
-
-static std::optional<unsigned> evalLaneExpr(const Value *V, unsigned Lane,
- unsigned WaveSize,
+static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
+ const GCNSubtarget &ST,
const DataLayout &DL,
unsigned Depth = 0) {
if (Depth > 16)
return std::nullopt;
- if (matchLaneID(V, WaveSize))
+ if (isThreadID(ST, V))
return Lane;
if (const auto *CI = dyn_cast<ConstantInt>(V))
return CI->getZExtValue();
- const auto *BO = dyn_cast<BinaryOperator>(V);
+ auto *BO = dyn_cast<BinaryOperator>(V);
if (!BO)
return std::nullopt;
- auto LHS = evalLaneExpr(BO->getOperand(0), Lane, WaveSize, DL, Depth + 1);
- auto RHS = evalLaneExpr(BO->getOperand(1), Lane, WaveSize, DL, Depth + 1);
+ auto LHS = evalLaneExpr(BO->getOperand(0), Lane, ST, DL, Depth + 1);
+ auto RHS = evalLaneExpr(BO->getOperand(1), Lane, ST, DL, Depth + 1);
if (!LHS || !RHS)
return std::nullopt;
@@ -779,12 +752,13 @@ static std::optional<unsigned> evalLaneExpr(const Value *V, unsigned Lane,
return CI->getZExtValue();
}
-static bool tryBuildShuffleMap(const Value *Index, unsigned WaveSize,
+static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
SmallVectorImpl<uint8_t> &Ids,
const DataLayout &DL) {
+ unsigned WaveSize = ST.getWavefrontSize();
Ids.resize(WaveSize);
for (unsigned Lane = 0; Lane < WaveSize; ++Lane) {
- auto Val = evalLaneExpr(Index, Lane, WaveSize, DL);
+ auto Val = evalLaneExpr(Index, Lane, ST, DL);
if (!Val || *Val >= WaveSize)
return false;
Ids[Lane] = static_cast<uint8_t>(*Val);
@@ -962,80 +936,70 @@ static Value *createPermlane64(IRBuilderBase &B, Value *Val, Module *M) {
/// Try to fold a wave_shuffle whose lane index is a constant function of the
/// lane ID into a hardware-specific lane permutation intrinsic.
-static std::optional<Instruction *>
-tryOptimizeWaveShuffle(InstCombiner &IC, IntrinsicInst &II,
- const GCNSubtarget &ST) {
+/// Returns the replacement Value, or nullptr if no pattern matched.
+static Value *tryOptimizeWaveShuffle(IRBuilderBase &B, IntrinsicInst &II,
+ const GCNSubtarget &ST,
+ const DataLayout &DL) {
if (II.getType()->getPrimitiveSizeInBits() != 32)
- return std::nullopt;
+ return nullptr;
if (!ST.isWaveSizeKnown())
- return std::nullopt;
+ return nullptr;
- unsigned WaveSize = ST.getWavefrontSize();
Value *Src = II.getArgOperand(0);
Value *Index = II.getArgOperand(1);
SmallVector<uint8_t, 64> Ids;
- if (!tryBuildShuffleMap(Index, WaveSize, Ids, IC.getDataLayout()))
- return std::nullopt;
+ if (!tryBuildShuffleMap(Index, ST, Ids, DL))
+ return nullptr;
- IRBuilderBase &B = IC.Builder;
Module *M = II.getModule();
- Value *Result = nullptr;
- // Try patterns in priority order: most specific first.
if (auto QP = matchQuadPermPattern(Ids)) {
if (ST.hasDPP())
- Result = createUpdateDpp(B, Src, *QP, M);
- else
- Result = createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, M);
+ return createUpdateDpp(B, Src, *QP, M);
+ return createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, M);
}
- if (!Result && ST.hasDPP() && matchHalfRowMirrorPattern(Ids))
- Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_HALF_MIRROR, M);
+ if (ST.hasDPP() && matchHalfRowMirrorPattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_HALF_MIRROR, M);
- if (!Result && ST.hasDPP() && matchFullRowMirrorPattern(Ids))
- Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_MIRROR, M);
+ if (ST.hasDPP() && matchFullRowMirrorPattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_MIRROR, M);
- if (!Result && ST.hasDPP()) {
+ if (ST.hasDPP()) {
if (auto Amt = matchRowRotatePattern(Ids))
- Result =
- createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1, M);
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1, M);
}
- if (!Result && ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
+ if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
if (auto Lane = matchRowSharePattern(Ids))
- Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane, M);
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane, M);
}
- if (!Result && ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
+ if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
if (auto Mask = matchRowXMaskPattern(Ids))
- Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask, M);
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask, M);
}
- if (!Result && ST.hasDPP8()) {
+ if (ST.hasDPP8()) {
if (auto Sel = matchHalfRowPermPattern(Ids))
- Result = createMovDpp8(B, Src, *Sel, M);
+ return createMovDpp8(B, Src, *Sel, M);
}
- if (!Result && ST.hasPermLaneX16() && isFullRowPattern(Ids)) {
+ if (ST.hasPermLaneX16() && isFullRowPattern(Ids)) {
auto [Lo, Hi] = computePermlane16Masks(Ids);
- Result = createPermlane16(B, Src, Lo, Hi, M);
+ return createPermlane16(B, Src, Lo, Hi, M);
}
// DS_SWIZZLE bitmask-mode fallback for targets without DPP8/permlane16.
- if (!Result) {
- if (auto Offset = matchHalfRowSharePattern(Ids))
- Result = createDsSwizzle(B, Src, *Offset, M);
- }
+ if (auto Offset = matchHalfRowSharePattern(Ids))
+ return createDsSwizzle(B, Src, *Offset, M);
- if (!Result && ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
- Result = createPermlane64(B, Src, M);
+ if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
+ return createPermlane64(B, Src, M);
- if (!Result)
- return std::nullopt;
-
- return IC.replaceInstUsesWith(II, Result);
+ return nullptr;
}
std::optional<Instruction *>
@@ -1882,8 +1846,9 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
if (ST->hasDPP())
if (auto R = tryWaveShuffleDPP(*ST, IC, II))
return R;
- if (auto R = tryOptimizeWaveShuffle(IC, II, *ST))
- return R;
+ if (Value *R =
+ tryOptimizeWaveShuffle(IC.Builder, II, *ST, IC.getDataLayout()))
+ return IC.replaceInstUsesWith(II, R);
break;
}
case Intrinsic::amdgcn_permlane64:
More information about the llvm-commits
mailing list