[llvm] [AMDGPU][InstCombine] Optimize constant shuffle patterns (PR #192246)
Aleksandar Spasojevic via llvm-commits
llvm-commits at lists.llvm.org
Tue May 12 04:59:13 PDT 2026
https://github.com/aleksandar-amd updated https://github.com/llvm/llvm-project/pull/192246
>From ec35d854a70971dcb7240f9da5ac7ffc81fa2c69 Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Tue, 14 Apr 2026 17:07:20 +0200
Subject: [PATCH 1/9] [AMDGPU][InstCombine] Match constant shuffle patterns to
specialized forms
Detect llvm.amdgcn.wave.shuffle intrinsics where the lane index is a
constant function of the lane ID and replace them with hardware-specific
intrinsics. This avoids the expensive ds_bpermute fallback, which
requires address computation and LDS memory access, by using
register-to-register lane permutation instructions that do not add LDS
pressure.
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 334 +++++++++++++++++-
.../AMDGPU/llvm.amdgcn.wave.shuffle.ll | 16 +-
.../AMDGPU/wave-shuffle-patterns.ll | 289 +++++++++++++++
3 files changed, 621 insertions(+), 18 deletions(-)
create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index c993e035b9b40..3559e230fad8a 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -17,7 +17,9 @@
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetTransformInfo.h"
#include "GCNSubtarget.h"
+#include "SIDefines.h"
#include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
@@ -718,6 +720,324 @@ GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
return nullptr;
}
+// Match llvm.amdgcn.mbcnt.lo / mbcnt.hi patterns that compute the lane ID.
+// WaveSize is needed because standalone mbcnt.lo returns lane % 32, which only
+// equals the lane ID on wave32. On wave64, the full mbcnt.hi(-1, mbcnt.lo(-1, 0))
+// chain is required.
+static bool matchLaneID(const Value *V, unsigned WaveSize) {
+ const auto *II = dyn_cast<IntrinsicInst>(V);
+ if (!II)
+ return false;
+
+ if (II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_hi) {
+ const auto *Mask = dyn_cast<ConstantInt>(II->getArgOperand(0));
+ if (Mask && Mask->getSExtValue() == -1)
+ return matchLaneID(II->getArgOperand(1), /*WaveSize=*/32);
+ return false;
+ }
+
+ if (II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo) {
+ if (WaveSize > 32)
+ return false;
+ const auto *Mask = dyn_cast<ConstantInt>(II->getArgOperand(0));
+ const auto *Base = dyn_cast<ConstantInt>(II->getArgOperand(1));
+ return Mask && Mask->getSExtValue() == -1 && Base && Base->isZero();
+ }
+
+ return false;
+}
+
+static std::optional<unsigned> evalLaneExpr(const Value *V, unsigned Lane,
+ unsigned WaveSize,
+ const DataLayout &DL,
+ unsigned Depth = 0) {
+ if (Depth > 16)
+ return std::nullopt;
+
+ if (matchLaneID(V, WaveSize))
+ return Lane;
+
+ if (const auto *CI = dyn_cast<ConstantInt>(V))
+ return CI->getZExtValue();
+
+ const auto *BO = dyn_cast<BinaryOperator>(V);
+ if (!BO)
+ return std::nullopt;
+
+ auto LHS = evalLaneExpr(BO->getOperand(0), Lane, WaveSize, DL, Depth + 1);
+ auto RHS = evalLaneExpr(BO->getOperand(1), Lane, WaveSize, DL, Depth + 1);
+ if (!LHS || !RHS)
+ return std::nullopt;
+
+ Type *Ty = BO->getType();
+ Constant *LC = ConstantInt::get(Ty, *LHS);
+ Constant *RC = ConstantInt::get(Ty, *RHS);
+ Constant *Result = ConstantFoldBinaryOpOperands(BO->getOpcode(), LC, RC, DL);
+ auto *CI = dyn_cast_or_null<ConstantInt>(Result);
+ if (!CI)
+ return std::nullopt;
+ return CI->getZExtValue();
+}
+
+static bool tryBuildShuffleMap(const Value *Index, unsigned WaveSize,
+ SmallVectorImpl<uint8_t> &Ids,
+ const DataLayout &DL) {
+ Ids.resize(WaveSize);
+ for (unsigned Lane = 0; Lane < WaveSize; ++Lane) {
+ auto Val = evalLaneExpr(Index, Lane, WaveSize, DL);
+ if (!Val || *Val >= WaveSize)
+ return false;
+ Ids[Lane] = static_cast<uint8_t>(*Val);
+ }
+ return true;
+}
+
+static bool isHalfRowPattern(ArrayRef<uint8_t> Ids) {
+ for (unsigned i = 0; i < 8; ++i)
+ if (Ids[i] >= 8)
+ return false;
+ for (unsigned i = 8, E = Ids.size(); i < E; ++i)
+ if (Ids[i] != Ids[i % 8] + (i & ~7u))
+ return false;
+ return true;
+}
+
+static bool isFullRowPattern(ArrayRef<uint8_t> Ids) {
+ for (unsigned i = 0; i < 16; ++i)
+ if (Ids[i] >= 16)
+ return false;
+ for (unsigned i = 16, E = Ids.size(); i < E; ++i)
+ if (Ids[i] != Ids[i % 16] + (i & ~15u))
+ return false;
+ return true;
+}
+
+static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
+ for (unsigned i = 0; i < 4; ++i)
+ if (Ids[i] >= 4)
+ return std::nullopt;
+ for (unsigned i = 4, E = Ids.size(); i < E; ++i)
+ if (Ids[i] != Ids[i % 4] + (i & ~3u))
+ return std::nullopt;
+ return (Ids[3] << 6) | (Ids[2] << 4) | (Ids[1] << 2) | Ids[0];
+}
+
+static bool matchHalfRowMirrorPattern(ArrayRef<uint8_t> Ids) {
+ if (!isHalfRowPattern(Ids))
+ return false;
+ for (unsigned j = 0; j < 8; ++j)
+ if (Ids[j] != 7 - j)
+ return false;
+ return true;
+}
+
+static bool matchFullRowMirrorPattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return false;
+ for (unsigned j = 0; j < 16; ++j)
+ if (Ids[j] != 15 - j)
+ return false;
+ return true;
+}
+
+static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return std::nullopt;
+ if (Ids[0] == 0 || Ids[15] == 15)
+ return std::nullopt;
+ for (unsigned j = 1; j < 16; ++j) {
+ int Diff = static_cast<int>(Ids[j]) - static_cast<int>(Ids[j - 1]);
+ if (Diff != 1 && Diff != -15)
+ return std::nullopt;
+ }
+ return 16u - Ids[0];
+}
+
+static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return std::nullopt;
+ for (unsigned j = 1; j < 16; ++j)
+ if (Ids[j] != Ids[0])
+ return std::nullopt;
+ return static_cast<unsigned>(Ids[0]);
+}
+
+static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return std::nullopt;
+ unsigned Mask = Ids[0];
+ if (Mask == 0)
+ return std::nullopt;
+ for (unsigned j = 0; j < 16; ++j)
+ if (Ids[j] != (Mask ^ j))
+ return std::nullopt;
+ return Mask;
+}
+
+static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
+ if (!isHalfRowPattern(Ids))
+ return std::nullopt;
+ unsigned Selector = 0;
+ for (unsigned j = 0; j < 8; ++j)
+ Selector |= static_cast<unsigned>(Ids[j] & 0x7) << (j * 3);
+ return Selector;
+}
+
+static std::pair<uint32_t, uint32_t>
+computePermlane16Masks(ArrayRef<uint8_t> Ids) {
+ uint32_t Lo = 0, Hi = 0;
+ for (unsigned j = 0; j < 8; ++j)
+ Lo |= static_cast<uint32_t>(Ids[j] & 0xF) << (j * 4);
+ for (unsigned j = 8; j < 16; ++j)
+ Hi |= static_cast<uint32_t>(Ids[j] & 0xF) << ((j - 8) * 4);
+ return {Lo, Hi};
+}
+
+static std::optional<unsigned> matchHalfRowSharePattern(ArrayRef<uint8_t> Ids) {
+ if (!isHalfRowPattern(Ids))
+ return std::nullopt;
+ for (unsigned j = 1; j < 8; ++j)
+ if (Ids[j] != Ids[0])
+ return std::nullopt;
+ // Bitmask-mode: AND=0x18 (keep group-select bits), OR=ids[0], XOR=0
+ return (static_cast<unsigned>(Ids[0]) << AMDGPU::Swizzle::BITMASK_OR_SHIFT) |
+ 0x18u;
+}
+
+static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
+ if (Ids.size() != 64)
+ return false;
+ for (unsigned j = 0; j < 64; ++j)
+ if (Ids[j] != (j ^ 32))
+ return false;
+ return true;
+}
+
+static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl,
+ Module *M) {
+ Type *Ty = Val->getType();
+ Function *F =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_update_dpp, {Ty});
+ return B.CreateCall(F, {PoisonValue::get(Ty), Val, B.getInt32(Ctrl),
+ B.getInt32(0xF), B.getInt32(0xF), B.getTrue()});
+}
+
+static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector,
+ Module *M) {
+ Type *Ty = Val->getType();
+ Function *F =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_mov_dpp8, {Ty});
+ return B.CreateCall(F, {Val, B.getInt32(Selector)});
+}
+
+static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
+ uint32_t Hi, Module *M) {
+ Type *Ty = Val->getType();
+ Function *F =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_permlane16, {Ty});
+ return B.CreateCall(F, {PoisonValue::get(Ty), Val, B.getInt32(Lo),
+ B.getInt32(Hi), B.getFalse(), B.getFalse()});
+}
+
+static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
+ Module *M) {
+ Type *OrigTy = Val->getType();
+ Value *Src = Val;
+ if (!OrigTy->isIntegerTy(32))
+ Src = B.CreateBitCast(Src, B.getInt32Ty());
+ Function *F =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_ds_swizzle);
+ Value *Result = B.CreateCall(F, {Src, B.getInt32(Offset)});
+ if (!OrigTy->isIntegerTy(32))
+ Result = B.CreateBitCast(Result, OrigTy);
+ return Result;
+}
+
+static Value *createPermlane64(IRBuilderBase &B, Value *Val, Module *M) {
+ Type *Ty = Val->getType();
+ Function *F =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_permlane64, {Ty});
+ return B.CreateCall(F, {Val});
+}
+
+/// Try to fold a wave_shuffle whose lane index is a constant function of the
+/// lane ID into a hardware-specific lane permutation intrinsic.
+static std::optional<Instruction *>
+tryOptimizeWaveShuffle(InstCombiner &IC, IntrinsicInst &II,
+ const GCNSubtarget &ST) {
+ if (II.getType()->getPrimitiveSizeInBits() != 32)
+ return std::nullopt;
+
+ if (!ST.isWaveSizeKnown())
+ return std::nullopt;
+
+ unsigned WaveSize = ST.getWavefrontSize();
+ Value *Src = II.getArgOperand(0);
+ Value *Index = II.getArgOperand(1);
+
+ SmallVector<uint8_t, 64> Ids;
+ if (!tryBuildShuffleMap(Index, WaveSize, Ids, IC.getDataLayout()))
+ return std::nullopt;
+
+ IRBuilderBase &B = IC.Builder;
+ Module *M = II.getModule();
+ Value *Result = nullptr;
+
+ // Try patterns in priority order: most specific first.
+ if (auto QP = matchQuadPermPattern(Ids)) {
+ if (ST.hasDPP())
+ Result = createUpdateDpp(B, Src, *QP, M);
+ else
+ Result = createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, M);
+ }
+
+ if (!Result && ST.hasDPP() && matchHalfRowMirrorPattern(Ids))
+ Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_HALF_MIRROR, M);
+
+ if (!Result && ST.hasDPP() && matchFullRowMirrorPattern(Ids))
+ Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_MIRROR, M);
+
+ if (!Result && ST.hasDPP()) {
+ if (auto Amt = matchRowRotatePattern(Ids))
+ Result =
+ createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1, M);
+ }
+
+ if (!Result && ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
+ if (auto Lane = matchRowSharePattern(Ids))
+ Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane, M);
+ }
+
+ if (!Result && ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
+ if (auto Mask = matchRowXMaskPattern(Ids))
+ Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask, M);
+ }
+
+ if (!Result && ST.hasDPP8()) {
+ if (auto Sel = matchHalfRowPermPattern(Ids))
+ Result = createMovDpp8(B, Src, *Sel, M);
+ }
+
+ if (!Result && ST.hasPermLaneX16() && isFullRowPattern(Ids)) {
+ auto [Lo, Hi] = computePermlane16Masks(Ids);
+ Result = createPermlane16(B, Src, Lo, Hi, M);
+ }
+
+ // DS_SWIZZLE bitmask-mode fallback for targets without DPP8/permlane16.
+ if (!Result) {
+ if (auto Offset = matchHalfRowSharePattern(Ids))
+ Result = createDsSwizzle(B, Src, *Offset, M);
+ }
+
+ if (!Result && ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
+ Result = createPermlane64(B, Src, M);
+
+ if (!Result)
+ return std::nullopt;
+
+ return IC.replaceInstUsesWith(II, Result);
+}
+
std::optional<Instruction *>
GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Intrinsic::ID IID = II.getIntrinsicID();
@@ -1559,6 +1879,14 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
return IC.replaceOperand(II, 0, PoisonValue::get(VDstIn->getType()));
}
+ case Intrinsic::amdgcn_wave_shuffle: {
+ if (ST->hasDPP())
+ if (auto R = tryWaveShuffleDPP(*ST, IC, II))
+ return R;
+ if (auto R = tryOptimizeWaveShuffle(IC, II, *ST))
+ return R;
+ break;
+ }
case Intrinsic::amdgcn_permlane64:
case Intrinsic::amdgcn_readfirstlane:
case Intrinsic::amdgcn_readlane:
@@ -1897,12 +2225,6 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
NewII->takeName(&II);
return IC.replaceInstUsesWith(II, NewII);
}
- case Intrinsic::amdgcn_wave_shuffle: {
- if (!ST->hasDPP())
- return std::nullopt;
-
- return tryWaveShuffleDPP(*ST, IC, II);
- }
}
if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
AMDGPU::getImageDimIntrinsicInfo(II.getIntrinsicID())) {
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
index 87fc259bdebff..ed415313b941c 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
@@ -8,15 +8,12 @@
define i32 @test_wave_shuffle_self_select(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_self_select(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-W32-NEXT: [[TID:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[TID]])
+; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 228, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT: ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_self_select(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-W64-NEXT: [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W64-NEXT: [[TID1:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[TID1]])
+; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 228, i32 15, i32 15, i1 true)
; CHECK-W64-NEXT: ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_self_select(
@@ -119,17 +116,12 @@ define i32 @test_wave_shuffle_dpp_row_share_7(i32 %val) {
define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[TID:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W32-NEXT: [[SHARE_7:%.*]] = or i32 [[TID]], 7
-; CHECK-W32-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL]], i32 16777215)
; CHECK-W32-NEXT: ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[LO:%.*]] = tail call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
-; CHECK-W64-NEXT: [[TID1:%.*]] = tail call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
-; CHECK-W64-NEXT: [[SHARE_7:%.*]] = or i32 [[TID1]], 7
-; CHECK-W64-NEXT: [[RES:%.*]] = tail call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL]], i32 [[SHARE_7]])
+; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL]], i32 16777215)
; CHECK-W64-NEXT: ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
new file mode 100644
index 0000000000000..fc42b90f72155
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
@@ -0,0 +1,289 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11-W64
+
+declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
+declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
+declare i32 @llvm.amdgcn.wave.shuffle.i32(i32, i32)
+declare float @llvm.amdgcn.wave.shuffle.f32(float, i32)
+
+; Quad perm: lane ^ 1 swaps adjacent pairs.
+define i32 @test_quad_perm_xor1_w32(i32 %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Quad perm: lane ^ 3 reverses within each quad.
+define i32 @test_quad_perm_xor3_w32(i32 %val) {
+; GFX11-LABEL: @test_quad_perm_xor3_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 27, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor3_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 3
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 3
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Half-row mirror: reverse within every 8-lane group.
+define i32 @test_half_row_mirror_w32(i32 %val) {
+; GFX11-LABEL: @test_half_row_mirror_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 321, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_half_row_mirror_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 7
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low3 = and i32 %lane, 7
+ %rev = sub i32 7, %low3
+ %high = and i32 %lane, -8
+ %idx = or i32 %high, %rev
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Full-row mirror: reverse within every 16-lane row.
+define i32 @test_full_row_mirror_w32(i32 %val) {
+; GFX11-LABEL: @test_full_row_mirror_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 320, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_full_row_mirror_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 15
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low4 = and i32 %lane, 15
+ %rev = sub i32 15, %low4
+ %high = and i32 %lane, -16
+ %idx = or i32 %high, %rev
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Row rotate right by 3 within each 16-lane row.
+define i32 @test_row_rotate_right3_w32(i32 %val) {
+; GFX11-LABEL: @test_row_rotate_right3_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 301, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_row_rotate_right3_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
+; GFX11-W64-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 15
+; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low4 = and i32 %lane, 15
+ %rot = add i32 %low4, 3
+ %wrapped = and i32 %rot, 15
+ %high = and i32 %lane, -16
+ %idx = or i32 %high, %wrapped
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Row share: broadcast lane 3 within each 16-lane row.
+define i32 @test_row_share3_w32(i32 %val) {
+; GFX11-LABEL: @test_row_share3_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 339, i32 15, i32 15, i1 false)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_row_share3_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %high = and i32 %lane, -16
+ %idx = or i32 %high, 3
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Row xmask: xor each lane with mask 5 within each 16-lane row.
+define i32 @test_row_xmask5_w32(i32 %val) {
+; GFX11-LABEL: @test_row_xmask5_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_row_xmask5_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 5
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low4 = and i32 %lane, 15
+ %xored = xor i32 %low4, 5
+ %high = and i32 %lane, -16
+ %idx = or i32 %high, %xored
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Half-wave swap (wave64 only): lane ^ 32.
+define i32 @test_half_wave_swap_w64(i32 %val) {
+; GFX11-LABEL: @test_half_wave_swap_w64(
+; GFX11-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LO]], 32
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_half_wave_swap_w64(
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane64.i32(i32 [[VAL:%.*]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %idx = xor i32 %lane, 32
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; DPP8: rotate left by 1 within each 8-lane group.
+define i32 @test_dpp8_rotate1_w32(i32 %val) {
+; GFX11-LABEL: @test_dpp8_rotate1_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.mov.dpp8.i32(i32 [[VAL:%.*]], i32 2054353)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_dpp8_rotate1_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 7
+; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 56
+; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low3 = and i32 %lane, 7
+ %rot = add i32 %low3, 1
+ %wrapped = and i32 %rot, 7
+ %high = and i32 %lane, -8
+ %idx = or i32 %high, %wrapped
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Permlane16: (lane * 3) & 15, a bijection on Z/16Z.
+define i32 @test_permlane16_mul3_w32(i32 %val) {
+; GFX11-LABEL: @test_permlane16_mul3_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane16.i32(i32 poison, i32 [[VAL:%.*]], i32 1392285232, i32 -629924168, i1 false, i1 false)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_permlane16_mul3_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
+; GFX11-W64-NEXT: [[PERM:%.*]] = and i32 [[MUL3]], 15
+; GFX11-W64-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %low4 = and i32 %lane, 15
+ %mul3 = mul i32 %low4, 3
+ %perm = and i32 %mul3, 15
+ %high = and i32 %lane, -16
+ %idx = or i32 %high, %perm
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Float type should work the same as i32.
+define float @test_quad_perm_xor1_float_w32(float %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_float_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.update.dpp.f32(float poison, float [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret float [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_float_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret float [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %result = call float @llvm.amdgcn.wave.shuffle.f32(float %val, i32 %idx)
+ ret float %result
+}
+
+; Negative: non-constant index, should not be optimized.
+define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
+; GFX11-LABEL: @test_nonconstant_shuffle_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_nonconstant_shuffle_w32(
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Identity shuffle: lane_id itself.
+define i32 @test_identity_shuffle_w32(i32 %val) {
+; GFX11-LABEL: @test_identity_shuffle_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 228, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_identity_shuffle_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %lane)
+ ret i32 %result
+}
+
+; Negative: index out of range (lane + 32 >= 32 for wave32).
+define i32 @test_out_of_range_w32(i32 %val) {
+; GFX11-LABEL: @test_out_of_range_w32(
+; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_out_of_range_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = add i32 %lane, 32
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
>From bb5a3cc16ab5bf68d2e9b25088db623ade5cbd4f Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Wed, 15 Apr 2026 16:40:25 +0200
Subject: [PATCH 2/9] Resolved comments
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 117 ++++++------------
1 file changed, 41 insertions(+), 76 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 3559e230fad8a..f06107f411c07 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -720,52 +720,25 @@ GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
return nullptr;
}
-// Match llvm.amdgcn.mbcnt.lo / mbcnt.hi patterns that compute the lane ID.
-// WaveSize is needed because standalone mbcnt.lo returns lane % 32, which only
-// equals the lane ID on wave32. On wave64, the full mbcnt.hi(-1, mbcnt.lo(-1, 0))
-// chain is required.
-static bool matchLaneID(const Value *V, unsigned WaveSize) {
- const auto *II = dyn_cast<IntrinsicInst>(V);
- if (!II)
- return false;
-
- if (II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_hi) {
- const auto *Mask = dyn_cast<ConstantInt>(II->getArgOperand(0));
- if (Mask && Mask->getSExtValue() == -1)
- return matchLaneID(II->getArgOperand(1), /*WaveSize=*/32);
- return false;
- }
-
- if (II->getIntrinsicID() == Intrinsic::amdgcn_mbcnt_lo) {
- if (WaveSize > 32)
- return false;
- const auto *Mask = dyn_cast<ConstantInt>(II->getArgOperand(0));
- const auto *Base = dyn_cast<ConstantInt>(II->getArgOperand(1));
- return Mask && Mask->getSExtValue() == -1 && Base && Base->isZero();
- }
-
- return false;
-}
-
-static std::optional<unsigned> evalLaneExpr(const Value *V, unsigned Lane,
- unsigned WaveSize,
+static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
+ const GCNSubtarget &ST,
const DataLayout &DL,
unsigned Depth = 0) {
if (Depth > 16)
return std::nullopt;
- if (matchLaneID(V, WaveSize))
+ if (isThreadID(ST, V))
return Lane;
if (const auto *CI = dyn_cast<ConstantInt>(V))
return CI->getZExtValue();
- const auto *BO = dyn_cast<BinaryOperator>(V);
+ auto *BO = dyn_cast<BinaryOperator>(V);
if (!BO)
return std::nullopt;
- auto LHS = evalLaneExpr(BO->getOperand(0), Lane, WaveSize, DL, Depth + 1);
- auto RHS = evalLaneExpr(BO->getOperand(1), Lane, WaveSize, DL, Depth + 1);
+ auto LHS = evalLaneExpr(BO->getOperand(0), Lane, ST, DL, Depth + 1);
+ auto RHS = evalLaneExpr(BO->getOperand(1), Lane, ST, DL, Depth + 1);
if (!LHS || !RHS)
return std::nullopt;
@@ -779,12 +752,13 @@ static std::optional<unsigned> evalLaneExpr(const Value *V, unsigned Lane,
return CI->getZExtValue();
}
-static bool tryBuildShuffleMap(const Value *Index, unsigned WaveSize,
+static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
SmallVectorImpl<uint8_t> &Ids,
const DataLayout &DL) {
+ unsigned WaveSize = ST.getWavefrontSize();
Ids.resize(WaveSize);
for (unsigned Lane = 0; Lane < WaveSize; ++Lane) {
- auto Val = evalLaneExpr(Index, Lane, WaveSize, DL);
+ auto Val = evalLaneExpr(Index, Lane, ST, DL);
if (!Val || *Val >= WaveSize)
return false;
Ids[Lane] = static_cast<uint8_t>(*Val);
@@ -962,80 +936,70 @@ static Value *createPermlane64(IRBuilderBase &B, Value *Val, Module *M) {
/// Try to fold a wave_shuffle whose lane index is a constant function of the
/// lane ID into a hardware-specific lane permutation intrinsic.
-static std::optional<Instruction *>
-tryOptimizeWaveShuffle(InstCombiner &IC, IntrinsicInst &II,
- const GCNSubtarget &ST) {
+/// Returns the replacement Value, or nullptr if no pattern matched.
+static Value *tryOptimizeWaveShuffle(IRBuilderBase &B, IntrinsicInst &II,
+ const GCNSubtarget &ST,
+ const DataLayout &DL) {
if (II.getType()->getPrimitiveSizeInBits() != 32)
- return std::nullopt;
+ return nullptr;
if (!ST.isWaveSizeKnown())
- return std::nullopt;
+ return nullptr;
- unsigned WaveSize = ST.getWavefrontSize();
Value *Src = II.getArgOperand(0);
Value *Index = II.getArgOperand(1);
SmallVector<uint8_t, 64> Ids;
- if (!tryBuildShuffleMap(Index, WaveSize, Ids, IC.getDataLayout()))
- return std::nullopt;
+ if (!tryBuildShuffleMap(Index, ST, Ids, DL))
+ return nullptr;
- IRBuilderBase &B = IC.Builder;
Module *M = II.getModule();
- Value *Result = nullptr;
- // Try patterns in priority order: most specific first.
if (auto QP = matchQuadPermPattern(Ids)) {
if (ST.hasDPP())
- Result = createUpdateDpp(B, Src, *QP, M);
- else
- Result = createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, M);
+ return createUpdateDpp(B, Src, *QP, M);
+ return createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, M);
}
- if (!Result && ST.hasDPP() && matchHalfRowMirrorPattern(Ids))
- Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_HALF_MIRROR, M);
+ if (ST.hasDPP() && matchHalfRowMirrorPattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_HALF_MIRROR, M);
- if (!Result && ST.hasDPP() && matchFullRowMirrorPattern(Ids))
- Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_MIRROR, M);
+ if (ST.hasDPP() && matchFullRowMirrorPattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_MIRROR, M);
- if (!Result && ST.hasDPP()) {
+ if (ST.hasDPP()) {
if (auto Amt = matchRowRotatePattern(Ids))
- Result =
- createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1, M);
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1, M);
}
- if (!Result && ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
+ if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
if (auto Lane = matchRowSharePattern(Ids))
- Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane, M);
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane, M);
}
- if (!Result && ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
+ if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
if (auto Mask = matchRowXMaskPattern(Ids))
- Result = createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask, M);
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask, M);
}
- if (!Result && ST.hasDPP8()) {
+ if (ST.hasDPP8()) {
if (auto Sel = matchHalfRowPermPattern(Ids))
- Result = createMovDpp8(B, Src, *Sel, M);
+ return createMovDpp8(B, Src, *Sel, M);
}
- if (!Result && ST.hasPermLaneX16() && isFullRowPattern(Ids)) {
+ if (ST.hasPermLaneX16() && isFullRowPattern(Ids)) {
auto [Lo, Hi] = computePermlane16Masks(Ids);
- Result = createPermlane16(B, Src, Lo, Hi, M);
+ return createPermlane16(B, Src, Lo, Hi, M);
}
// DS_SWIZZLE bitmask-mode fallback for targets without DPP8/permlane16.
- if (!Result) {
- if (auto Offset = matchHalfRowSharePattern(Ids))
- Result = createDsSwizzle(B, Src, *Offset, M);
- }
+ if (auto Offset = matchHalfRowSharePattern(Ids))
+ return createDsSwizzle(B, Src, *Offset, M);
- if (!Result && ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
- Result = createPermlane64(B, Src, M);
+ if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
+ return createPermlane64(B, Src, M);
- if (!Result)
- return std::nullopt;
-
- return IC.replaceInstUsesWith(II, Result);
+ return nullptr;
}
std::optional<Instruction *>
@@ -1883,8 +1847,9 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
if (ST->hasDPP())
if (auto R = tryWaveShuffleDPP(*ST, IC, II))
return R;
- if (auto R = tryOptimizeWaveShuffle(IC, II, *ST))
- return R;
+ if (Value *R =
+ tryOptimizeWaveShuffle(IC.Builder, II, *ST, IC.getDataLayout()))
+ return IC.replaceInstUsesWith(II, R);
break;
}
case Intrinsic::amdgcn_permlane64:
>From 372a41e0699e531d97df9242b9c9b1e5ad2c60d6 Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Fri, 24 Apr 2026 17:54:47 +0200
Subject: [PATCH 3/9] Resolved comments 2
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 239 +++++++++---------
.../AMDGPU/wave-shuffle-patterns.ll | 121 +++++++++
2 files changed, 237 insertions(+), 123 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index f06107f411c07..6ef8974c5bf07 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -730,23 +730,26 @@ static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
if (isThreadID(ST, V))
return Lane;
- if (const auto *CI = dyn_cast<ConstantInt>(V))
+ if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
return CI->getZExtValue();
- auto *BO = dyn_cast<BinaryOperator>(V);
+ const BinaryOperator *BO = dyn_cast<BinaryOperator>(V);
if (!BO)
return std::nullopt;
- auto LHS = evalLaneExpr(BO->getOperand(0), Lane, ST, DL, Depth + 1);
- auto RHS = evalLaneExpr(BO->getOperand(1), Lane, ST, DL, Depth + 1);
- if (!LHS || !RHS)
+ std::optional<unsigned> LHS =
+ evalLaneExpr(BO->getOperand(0), Lane, ST, DL, Depth + 1);
+ if (!LHS)
+ return std::nullopt;
+ std::optional<unsigned> RHS =
+ evalLaneExpr(BO->getOperand(1), Lane, ST, DL, Depth + 1);
+ if (!RHS)
return std::nullopt;
Type *Ty = BO->getType();
- Constant *LC = ConstantInt::get(Ty, *LHS);
- Constant *RC = ConstantInt::get(Ty, *RHS);
- Constant *Result = ConstantFoldBinaryOpOperands(BO->getOpcode(), LC, RC, DL);
- auto *CI = dyn_cast_or_null<ConstantInt>(Result);
+ Constant *Ops[] = {ConstantInt::get(Ty, *LHS), ConstantInt::get(Ty, *RHS)};
+ auto *CI =
+ dyn_cast_or_null<ConstantInt>(ConstantFoldInstOperands(BO, Ops, DL));
if (!CI)
return std::nullopt;
return CI->getZExtValue();
@@ -766,53 +769,37 @@ static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
return true;
}
-static bool isHalfRowPattern(ArrayRef<uint8_t> Ids) {
- for (unsigned i = 0; i < 8; ++i)
- if (Ids[i] >= 8)
+template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
+ for (unsigned i = 0; i < N; ++i)
+ if (Ids[i] >= N)
return false;
- for (unsigned i = 8, E = Ids.size(); i < E; ++i)
- if (Ids[i] != Ids[i % 8] + (i & ~7u))
+ for (unsigned i = N, E = Ids.size(); i < E; ++i)
+ if (Ids[i] != Ids[i % N] + (i & ~(N - 1)))
return false;
return true;
}
-static bool isFullRowPattern(ArrayRef<uint8_t> Ids) {
- for (unsigned i = 0; i < 16; ++i)
- if (Ids[i] >= 16)
- return false;
- for (unsigned i = 16, E = Ids.size(); i < E; ++i)
- if (Ids[i] != Ids[i % 16] + (i & ~15u))
- return false;
- return true;
-}
+static constexpr auto isQuadPattern = isRowPattern<4>;
+static constexpr auto isHalfRowPattern = isRowPattern<8>;
+static constexpr auto isFullRowPattern = isRowPattern<16>;
static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
- for (unsigned i = 0; i < 4; ++i)
- if (Ids[i] >= 4)
- return std::nullopt;
- for (unsigned i = 4, E = Ids.size(); i < E; ++i)
- if (Ids[i] != Ids[i % 4] + (i & ~3u))
- return std::nullopt;
+ if (!isQuadPattern(Ids))
+ return std::nullopt;
return (Ids[3] << 6) | (Ids[2] << 4) | (Ids[1] << 2) | Ids[0];
}
-static bool matchHalfRowMirrorPattern(ArrayRef<uint8_t> Ids) {
- if (!isHalfRowPattern(Ids))
+template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
+ if (!isRowPattern<N>(Ids))
return false;
- for (unsigned j = 0; j < 8; ++j)
- if (Ids[j] != 7 - j)
+ for (unsigned j = 0; j < N; ++j)
+ if (Ids[j] != (N - 1) - j)
return false;
return true;
}
-static bool matchFullRowMirrorPattern(ArrayRef<uint8_t> Ids) {
- if (!isFullRowPattern(Ids))
- return false;
- for (unsigned j = 0; j < 16; ++j)
- if (Ids[j] != 15 - j)
- return false;
- return true;
-}
+static constexpr auto matchHalfRowMirrorPattern = matchMirrorPattern<8>;
+static constexpr auto matchFullRowMirrorPattern = matchMirrorPattern<16>;
static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
if (!isFullRowPattern(Ids))
@@ -887,121 +874,129 @@ static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
return true;
}
-static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl,
- Module *M) {
+static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
Type *Ty = Val->getType();
- Function *F =
- Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_update_dpp, {Ty});
- return B.CreateCall(F, {PoisonValue::get(Ty), Val, B.getInt32(Ctrl),
- B.getInt32(0xF), B.getInt32(0xF), B.getTrue()});
+ return B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, {Ty},
+ {PoisonValue::get(Ty), Val, B.getInt32(Ctrl),
+ B.getInt32(0xF), B.getInt32(0xF), B.getTrue()});
}
-static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector,
- Module *M) {
- Type *Ty = Val->getType();
- Function *F =
- Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_mov_dpp8, {Ty});
- return B.CreateCall(F, {Val, B.getInt32(Selector)});
+static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector) {
+ return B.CreateIntrinsic(Intrinsic::amdgcn_mov_dpp8, {Val->getType()},
+ {Val, B.getInt32(Selector)});
}
static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
- uint32_t Hi, Module *M) {
+ uint32_t Hi) {
Type *Ty = Val->getType();
- Function *F =
- Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_permlane16, {Ty});
- return B.CreateCall(F, {PoisonValue::get(Ty), Val, B.getInt32(Lo),
- B.getInt32(Hi), B.getFalse(), B.getFalse()});
+ return B.CreateIntrinsic(Intrinsic::amdgcn_permlane16, {Ty},
+ {PoisonValue::get(Ty), Val, B.getInt32(Lo),
+ B.getInt32(Hi), B.getFalse(), B.getFalse()});
}
-static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
- Module *M) {
+static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset) {
Type *OrigTy = Val->getType();
+ assert(OrigTy->getPrimitiveSizeInBits() == 32 &&
+ "ds_swizzle only supports 32-bit operands");
Value *Src = Val;
if (!OrigTy->isIntegerTy(32))
Src = B.CreateBitCast(Src, B.getInt32Ty());
- Function *F =
- Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_ds_swizzle);
- Value *Result = B.CreateCall(F, {Src, B.getInt32(Offset)});
+ Value *Result = B.CreateIntrinsic(Intrinsic::amdgcn_ds_swizzle, {},
+ {Src, B.getInt32(Offset)});
if (!OrigTy->isIntegerTy(32))
Result = B.CreateBitCast(Result, OrigTy);
return Result;
}
-static Value *createPermlane64(IRBuilderBase &B, Value *Val, Module *M) {
- Type *Ty = Val->getType();
- Function *F =
- Intrinsic::getOrInsertDeclaration(M, Intrinsic::amdgcn_permlane64, {Ty});
- return B.CreateCall(F, {Val});
+static Value *createPermlane64(IRBuilderBase &B, Value *Val) {
+ return B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {Val->getType()},
+ {Val});
}
-/// Try to fold a wave_shuffle whose lane index is a constant function of the
-/// lane ID into a hardware-specific lane permutation intrinsic.
-/// Returns the replacement Value, or nullptr if no pattern matched.
-static Value *tryOptimizeWaveShuffle(IRBuilderBase &B, IntrinsicInst &II,
- const GCNSubtarget &ST,
- const DataLayout &DL) {
- if (II.getType()->getPrimitiveSizeInBits() != 32)
- return nullptr;
-
- if (!ST.isWaveSizeKnown())
- return nullptr;
-
- Value *Src = II.getArgOperand(0);
- Value *Index = II.getArgOperand(1);
-
- SmallVector<uint8_t, 64> Ids;
- if (!tryBuildShuffleMap(Index, ST, Ids, DL))
- return nullptr;
-
- Module *M = II.getModule();
-
- if (auto QP = matchQuadPermPattern(Ids)) {
+/// Given a shuffle map, try to emit the best hardware intrinsic.
+static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
+ ArrayRef<uint8_t> Ids,
+ const GCNSubtarget &ST) {
+ if (std::optional<unsigned> QP = matchQuadPermPattern(Ids)) {
if (ST.hasDPP())
- return createUpdateDpp(B, Src, *QP, M);
- return createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, M);
+ return createUpdateDpp(B, Src, *QP);
+ return createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP);
}
- if (ST.hasDPP() && matchHalfRowMirrorPattern(Ids))
- return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_HALF_MIRROR, M);
-
- if (ST.hasDPP() && matchFullRowMirrorPattern(Ids))
- return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_MIRROR, M);
-
if (ST.hasDPP()) {
- if (auto Amt = matchRowRotatePattern(Ids))
- return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1, M);
+ if (matchHalfRowMirrorPattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_HALF_MIRROR);
+ if (matchFullRowMirrorPattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_MIRROR);
+ if (std::optional<unsigned> Amt = matchRowRotatePattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1);
}
- if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
- if (auto Lane = matchRowSharePattern(Ids))
- return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane, M);
- }
-
- if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
- if (auto Mask = matchRowXMaskPattern(Ids))
- return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask, M);
+ if (ST.hasDPP() && ST.hasGFX10Insts()) {
+ if (std::optional<unsigned> Lane = matchRowSharePattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane);
+ if (std::optional<unsigned> Mask = matchRowXMaskPattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask);
}
if (ST.hasDPP8()) {
- if (auto Sel = matchHalfRowPermPattern(Ids))
- return createMovDpp8(B, Src, *Sel, M);
+ if (std::optional<unsigned> Sel = matchHalfRowPermPattern(Ids))
+ return createMovDpp8(B, Src, *Sel);
}
if (ST.hasPermLaneX16() && isFullRowPattern(Ids)) {
auto [Lo, Hi] = computePermlane16Masks(Ids);
- return createPermlane16(B, Src, Lo, Hi, M);
+ return createPermlane16(B, Src, Lo, Hi);
}
// DS_SWIZZLE bitmask-mode fallback for targets without DPP8/permlane16.
- if (auto Offset = matchHalfRowSharePattern(Ids))
- return createDsSwizzle(B, Src, *Offset, M);
+ if (std::optional<unsigned> Offset = matchHalfRowSharePattern(Ids))
+ return createDsSwizzle(B, Src, *Offset);
if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
- return createPermlane64(B, Src, M);
+ return createPermlane64(B, Src);
return nullptr;
}
+/// Try to fold a wave_shuffle/ds_bpermute whose lane index is a constant
+/// function of the lane ID into a hardware-specific lane permutation intrinsic.
+static std::optional<Instruction *>
+tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II,
+ const GCNSubtarget &ST) {
+ if (II.getType()->getPrimitiveSizeInBits() != 32)
+ return std::nullopt;
+
+ if (!ST.isWaveSizeKnown())
+ return std::nullopt;
+
+ unsigned WaveSize = ST.getWavefrontSize();
+ bool IsBpermute = II.getIntrinsicID() == Intrinsic::amdgcn_ds_bpermute;
+ Value *Src = II.getArgOperand(IsBpermute ? 1 : 0);
+ Value *Index = II.getArgOperand(IsBpermute ? 0 : 1);
+
+ SmallVector<uint8_t, 64> Ids;
+ if (IsBpermute) {
+ Ids.resize(WaveSize);
+ for (unsigned Lane = 0; Lane < WaveSize; ++Lane) {
+ std::optional<unsigned> Val =
+ evalLaneExpr(Index, Lane, ST, IC.getDataLayout());
+ if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
+ return std::nullopt;
+ Ids[Lane] = static_cast<uint8_t>(*Val >> 2);
+ }
+ } else {
+ if (!tryBuildShuffleMap(Index, ST, Ids, IC.getDataLayout()))
+ return std::nullopt;
+ }
+
+ Value *Result = matchShuffleToHWIntrinsic(IC.Builder, Src, Ids, ST);
+ if (!Result)
+ return std::nullopt;
+
+ return IC.replaceInstUsesWith(II, Result);
+}
+
std::optional<Instruction *>
GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Intrinsic::ID IID = II.getIntrinsicID();
@@ -1845,12 +1840,9 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
}
case Intrinsic::amdgcn_wave_shuffle: {
if (ST->hasDPP())
- if (auto R = tryWaveShuffleDPP(*ST, IC, II))
+ if (std::optional<Instruction *> R = tryWaveShuffleDPP(*ST, IC, II))
return R;
- if (Value *R =
- tryOptimizeWaveShuffle(IC.Builder, II, *ST, IC.getDataLayout()))
- return IC.replaceInstUsesWith(II, R);
- break;
+ return tryOptimizeShufflePattern(IC, II, *ST);
}
case Intrinsic::amdgcn_permlane64:
case Intrinsic::amdgcn_readfirstlane:
@@ -1882,10 +1874,11 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
}
}
- if (IID != Intrinsic::amdgcn_ds_bpermute) {
- if (Instruction *Res = hoistLaneIntrinsicThroughOperand(IC, II))
- return Res;
- }
+ if (IID == Intrinsic::amdgcn_ds_bpermute)
+ return tryOptimizeShufflePattern(IC, II, *ST);
+
+ if (Instruction *Res = hoistLaneIntrinsicThroughOperand(IC, II))
+ return Res;
return std::nullopt;
}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
index fc42b90f72155..88bd374c74752 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
@@ -1,11 +1,13 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11
; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=+wavefrontsize64 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX11-W64
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=instcombine -S < %s | FileCheck %s --check-prefixes=GFX9
declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
declare i32 @llvm.amdgcn.wave.shuffle.i32(i32, i32)
declare float @llvm.amdgcn.wave.shuffle.f32(float, i32)
+declare i32 @llvm.amdgcn.ds.bpermute(i32, i32)
; Quad perm: lane ^ 1 swaps adjacent pairs.
define i32 @test_quad_perm_xor1_w32(i32 %val) {
@@ -18,6 +20,12 @@ define i32 @test_quad_perm_xor1_w32(i32 %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
@@ -36,6 +44,12 @@ define i32 @test_quad_perm_xor3_w32(i32 %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 3
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor3_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 3
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 3
@@ -54,6 +68,12 @@ define i32 @test_half_row_mirror_w32(i32 %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 7
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_half_row_mirror_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 7
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low3 = and i32 %lane, 7
@@ -75,6 +95,12 @@ define i32 @test_full_row_mirror_w32(i32 %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 15
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_full_row_mirror_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 15
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low4 = and i32 %lane, 15
@@ -99,6 +125,15 @@ define i32 @test_row_rotate_right3_w32(i32 %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_row_rotate_right3_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 3
+; GFX9-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 15
+; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low4 = and i32 %lane, 15
@@ -122,6 +157,13 @@ define i32 @test_row_share3_w32(i32 %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_row_share3_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], 3
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%high = and i32 %lane, -16
@@ -141,6 +183,12 @@ define i32 @test_row_xmask5_w32(i32 %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 5
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_row_xmask5_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 5
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low4 = and i32 %lane, 15
@@ -162,6 +210,13 @@ define i32 @test_half_wave_swap_w64(i32 %val) {
; GFX11-W64-LABEL: @test_half_wave_swap_w64(
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlane64.i32(i32 [[VAL:%.*]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_half_wave_swap_w64(
+; GFX9-NEXT: [[LO:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 65) i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 [[LO]])
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 32
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
@@ -184,6 +239,15 @@ define i32 @test_dpp8_rotate1_w32(i32 %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_dpp8_rotate1_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[ROT:%.*]] = add nuw nsw i32 [[LANE]], 1
+; GFX9-NEXT: [[WRAPPED:%.*]] = and i32 [[ROT]], 7
+; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 56
+; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[WRAPPED]]
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low3 = and i32 %lane, 7
@@ -209,6 +273,15 @@ define i32 @test_permlane16_mul3_w32(i32 %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_permlane16_mul3_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[MUL3:%.*]] = mul nuw nsw i32 [[LANE]], 3
+; GFX9-NEXT: [[PERM:%.*]] = and i32 [[MUL3]], 15
+; GFX9-NEXT: [[HIGH:%.*]] = and i32 [[LANE]], 48
+; GFX9-NEXT: [[IDX:%.*]] = or disjoint i32 [[HIGH]], [[PERM]]
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%low4 = and i32 %lane, 15
@@ -231,6 +304,12 @@ define float @test_quad_perm_xor1_float_w32(float %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
; GFX11-W64-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret float [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_float_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT: [[RESULT:%.*]] = call float @llvm.amdgcn.wave.shuffle.f32(float [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret float [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = xor i32 %lane, 1
@@ -247,6 +326,10 @@ define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
; GFX11-W64-LABEL: @test_nonconstant_shuffle_w32(
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_nonconstant_shuffle_w32(
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX:%.*]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
@@ -262,6 +345,11 @@ define i32 @test_identity_shuffle_w32(i32 %val) {
; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_identity_shuffle_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[LANE]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %lane)
@@ -281,9 +369,42 @@ define i32 @test_out_of_range_w32(i32 %val) {
; GFX11-W64-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_out_of_range_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = add nuw nsw i32 [[LANE]], 32
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
;
%lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
%idx = add i32 %lane, 32
%result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
ret i32 %result
}
+
+; ds_bpermute with a constant shuffle pattern (quad perm via byte address).
+define i32 @test_bpermute_quad_xor1_w32(i32 %val) {
+; GFX11-LABEL: @test_bpermute_quad_xor1_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_bpermute_quad_xor1_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = shl nuw nsw i32 [[LANE]], 2
+; GFX11-W64-NEXT: [[ADDR:%.*]] = xor i32 [[IDX]], 4
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_bpermute_quad_xor1_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = shl nuw nsw i32 [[LANE]], 2
+; GFX9-NEXT: [[ADDR:%.*]] = xor i32 [[IDX]], 4
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.bpermute(i32 [[ADDR]], i32 [[VAL:%.*]])
+; GFX9-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %addr = shl i32 %idx, 2
+ %result = call i32 @llvm.amdgcn.ds.bpermute(i32 %addr, i32 %val)
+ ret i32 %result
+}
>From d7218a978e55b545d1cda7f93ec5b94808b3e5bc Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Tue, 28 Apr 2026 12:04:59 +0200
Subject: [PATCH 4/9] Resolved comments 3
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 36 +++++++++++--------
.../AMDGPU/wave-shuffle-patterns.ll | 26 ++++++++++++++
2 files changed, 48 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 6ef8974c5bf07..4988fde969728 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -20,6 +20,7 @@
#include "SIDefines.h"
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/Analysis/ConstantFolding.h"
+#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
@@ -724,7 +725,7 @@ static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
const GCNSubtarget &ST,
const DataLayout &DL,
unsigned Depth = 0) {
- if (Depth > 16)
+ if (Depth >= MaxAnalysisRecursionDepth)
return std::nullopt;
if (isThreadID(ST, V))
@@ -894,17 +895,23 @@ static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
B.getInt32(Hi), B.getFalse(), B.getFalse()});
}
-static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset) {
+static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
+ const DataLayout &DL) {
Type *OrigTy = Val->getType();
- assert(OrigTy->getPrimitiveSizeInBits() == 32 &&
+ assert(DL.getTypeSizeInBits(OrigTy) == 32 &&
"ds_swizzle only supports 32-bit operands");
+ IntegerType *I32Ty = B.getInt32Ty();
Value *Src = Val;
- if (!OrigTy->isIntegerTy(32))
- Src = B.CreateBitCast(Src, B.getInt32Ty());
+ if (OrigTy->isPointerTy())
+ Src = B.CreatePtrToInt(Src, I32Ty);
+ else if (!OrigTy->isIntegerTy(32))
+ Src = B.CreateBitCast(Src, I32Ty);
Value *Result = B.CreateIntrinsic(Intrinsic::amdgcn_ds_swizzle, {},
{Src, B.getInt32(Offset)});
+ if (OrigTy->isPointerTy())
+ return B.CreateIntToPtr(Result, OrigTy);
if (!OrigTy->isIntegerTy(32))
- Result = B.CreateBitCast(Result, OrigTy);
+ return B.CreateBitCast(Result, OrigTy);
return Result;
}
@@ -916,11 +923,12 @@ static Value *createPermlane64(IRBuilderBase &B, Value *Val) {
/// Given a shuffle map, try to emit the best hardware intrinsic.
static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
ArrayRef<uint8_t> Ids,
- const GCNSubtarget &ST) {
+ const GCNSubtarget &ST,
+ const DataLayout &DL) {
if (std::optional<unsigned> QP = matchQuadPermPattern(Ids)) {
if (ST.hasDPP())
return createUpdateDpp(B, Src, *QP);
- return createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP);
+ return createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, DL);
}
if (ST.hasDPP()) {
@@ -951,7 +959,7 @@ static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
// DS_SWIZZLE bitmask-mode fallback for targets without DPP8/permlane16.
if (std::optional<unsigned> Offset = matchHalfRowSharePattern(Ids))
- return createDsSwizzle(B, Src, *Offset);
+ return createDsSwizzle(B, Src, *Offset, DL);
if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
return createPermlane64(B, Src);
@@ -964,7 +972,8 @@ static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
static std::optional<Instruction *>
tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II,
const GCNSubtarget &ST) {
- if (II.getType()->getPrimitiveSizeInBits() != 32)
+ const DataLayout &DL = IC.getDataLayout();
+ if (DL.getTypeSizeInBits(II.getType()) != 32)
return std::nullopt;
if (!ST.isWaveSizeKnown())
@@ -979,18 +988,17 @@ tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II,
if (IsBpermute) {
Ids.resize(WaveSize);
for (unsigned Lane = 0; Lane < WaveSize; ++Lane) {
- std::optional<unsigned> Val =
- evalLaneExpr(Index, Lane, ST, IC.getDataLayout());
+ std::optional<unsigned> Val = evalLaneExpr(Index, Lane, ST, DL);
if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
return std::nullopt;
Ids[Lane] = static_cast<uint8_t>(*Val >> 2);
}
} else {
- if (!tryBuildShuffleMap(Index, ST, Ids, IC.getDataLayout()))
+ if (!tryBuildShuffleMap(Index, ST, Ids, DL))
return std::nullopt;
}
- Value *Result = matchShuffleToHWIntrinsic(IC.Builder, Src, Ids, ST);
+ Value *Result = matchShuffleToHWIntrinsic(IC.Builder, Src, Ids, ST, DL);
if (!Result)
return std::nullopt;
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
index 88bd374c74752..fa75823f6c9f5 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
@@ -7,6 +7,7 @@ declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
declare i32 @llvm.amdgcn.wave.shuffle.i32(i32, i32)
declare float @llvm.amdgcn.wave.shuffle.f32(float, i32)
+declare ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3), i32)
declare i32 @llvm.amdgcn.ds.bpermute(i32, i32)
; Quad perm: lane ^ 1 swaps adjacent pairs.
@@ -317,6 +318,31 @@ define float @test_quad_perm_xor1_float_w32(float %val) {
ret float %result
}
+; LDS (32-bit) pointer should optimize the same as i32: the gate must not
+; reject 32-bit pointer types via Type::getPrimitiveSizeInBits().
+define ptr addrspace(3) @test_quad_perm_xor1_ptr3_w32(ptr addrspace(3) %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_ptr3_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.update.dpp.p3(ptr addrspace(3) poison, ptr addrspace(3) [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret ptr addrspace(3) [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_ptr3_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret ptr addrspace(3) [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_ptr3_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT: [[RESULT:%.*]] = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret ptr addrspace(3) [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %result = call ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3) %val, i32 %idx)
+ ret ptr addrspace(3) %result
+}
+
; Negative: non-constant index, should not be optimized.
define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
; GFX11-LABEL: @test_nonconstant_shuffle_w32(
>From fef237f0922e3cfee09c64e788d41d7e494e0575 Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Wed, 29 Apr 2026 14:15:24 +0200
Subject: [PATCH 5/9] Added support for handling v_permlanex16
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 127 +++++++-----------
.../AMDGPU/llvm.amdgcn.wave.shuffle.ll | 16 +--
.../AMDGPU/wave-shuffle-patterns.ll | 74 +++++++++-
3 files changed, 131 insertions(+), 86 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 4988fde969728..be98fc7595146 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -575,70 +575,6 @@ static bool isThreadID(const GCNSubtarget &ST, Value *V) {
return false;
}
-// Attempt to capture situations where the index argument matches
-// a DPP pattern, and convert to a DPP-based mov
-static std::optional<Instruction *>
-tryWaveShuffleDPP(const GCNSubtarget &ST, InstCombiner &IC, IntrinsicInst &II) {
- Value *Val = II.getArgOperand(0);
- Value *Idx = II.getArgOperand(1);
- auto &B = IC.Builder;
-
- // DPP16 Row Share requires known wave size, architecture support
- if (!ST.isWaveSizeKnown() || !ST.hasDPPRowShare())
- return std::nullopt;
-
- Value *Tid;
- uint64_t Mask;
- uint64_t RowIdx;
- bool CanDPP16RowShare = false;
-
- // wave32 requires Mask & 0x1F == 0x10
- // wave64 requires Mask & 0x3F == 0x30
- uint64_t MaskCheck = (1UL << ST.getWavefrontSizeLog2()) - 1;
- uint64_t MaskTarget = MaskCheck & 0xF0;
-
- // DPP16 Row Share 0: Idx = Tid & Mask
- auto RowShare0Pred = m_And(m_Value(Tid), m_ConstantInt(Mask));
-
- // DPP16 Row Share (0 < Row < 15): Idx = (Tid & Mask) | RowIdx
- auto RowSharePred =
- m_Or(m_And(m_Value(Tid), m_ConstantInt(Mask)), m_ConstantInt(RowIdx));
-
- // DPP16 Row Share 15: Idx = Tid | 0xF
- auto RowShare15Pred = m_Or(m_Value(Tid), m_ConstantInt<0xF>());
-
- if (match(Idx, RowShare0Pred) && isThreadID(ST, Tid)) {
- if ((Mask & MaskCheck) != MaskTarget)
- return std::nullopt;
-
- RowIdx = 0;
- CanDPP16RowShare = true;
- } else if (match(Idx, RowSharePred) && isThreadID(ST, Tid) && RowIdx < 15 &&
- RowIdx > 0) {
- if ((Mask & MaskCheck) != MaskTarget)
- return std::nullopt;
-
- CanDPP16RowShare = true;
- } else if (match(Idx, RowShare15Pred) && isThreadID(ST, Tid)) {
- RowIdx = 15;
- CanDPP16RowShare = true;
- }
-
- if (CanDPP16RowShare) {
- CallInst *UpdateDPP =
- B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, Val->getType(),
- {PoisonValue::get(Val->getType()), Val,
- B.getInt32(AMDGPU::DPP::ROW_SHARE0 | RowIdx),
- B.getInt32(0xF), B.getInt32(0xF), B.getFalse()});
- UpdateDPP->takeName(&II);
- UpdateDPP->copyMetadata(II);
- return IC.replaceInstUsesWith(II, UpdateDPP);
- }
-
- // No valid DPP detected
- return std::nullopt;
-}
-
Instruction *
GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
IntrinsicInst &II) const {
@@ -770,14 +706,23 @@ static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
return true;
}
+/// Lanes are partitioned into groups of Period; each group is a translated
+/// copy of the first: Ids[i] = Ids[i % Period] + (i & ~(Period - 1)).
+template <unsigned Period>
+static bool hasPeriodicLayout(ArrayRef<uint8_t> Ids) {
+ static_assert(Period > 0 && (Period & (Period - 1)) == 0,
+ "Period must be a power of two");
+ for (unsigned i = Period, E = Ids.size(); i < E; ++i)
+ if (Ids[i] != Ids[i % Period] + (i & ~(Period - 1)))
+ return false;
+ return true;
+}
+
template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
for (unsigned i = 0; i < N; ++i)
if (Ids[i] >= N)
return false;
- for (unsigned i = N, E = Ids.size(); i < E; ++i)
- if (Ids[i] != Ids[i % N] + (i & ~(N - 1)))
- return false;
- return true;
+ return hasPeriodicLayout<N>(Ids);
}
static constexpr auto isQuadPattern = isRowPattern<4>;
@@ -875,6 +820,19 @@ static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
return true;
}
+// Detects a cross-row shuffle pattern suitable for v_permlanex16
+static bool isCrossRowPattern(ArrayRef<uint8_t> Ids) {
+ if (Ids.size() < 32 || !hasPeriodicLayout<32>(Ids))
+ return false;
+ for (unsigned j = 0; j < 16; ++j) {
+ if (Ids[j] < 16 || Ids[j] >= 32)
+ return false;
+ if (Ids[j + 16] != Ids[j] - 16)
+ return false;
+ }
+ return true;
+}
+
static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
Type *Ty = Val->getType();
return B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, {Ty},
@@ -895,6 +853,14 @@ static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
B.getInt32(Hi), B.getFalse(), B.getFalse()});
}
+static Value *createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo,
+ uint32_t Hi) {
+ Type *Ty = Val->getType();
+ return B.CreateIntrinsic(Intrinsic::amdgcn_permlanex16, {Ty},
+ {PoisonValue::get(Ty), Val, B.getInt32(Lo),
+ B.getInt32(Hi), B.getFalse(), B.getFalse()});
+}
+
static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
const DataLayout &DL) {
Type *OrigTy = Val->getType();
@@ -940,9 +906,13 @@ static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1);
}
- if (ST.hasDPP() && ST.hasGFX10Insts()) {
+ // row_share is supported on GFX90A and GFX10+; row_xmask is GFX10+ only.
+ if (ST.hasDPPRowShare()) {
if (std::optional<unsigned> Lane = matchRowSharePattern(Ids))
return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane);
+ }
+
+ if (ST.hasDPP() && ST.hasGFX10Insts()) {
if (std::optional<unsigned> Mask = matchRowXMaskPattern(Ids))
return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask);
}
@@ -952,9 +922,16 @@ static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
return createMovDpp8(B, Src, *Sel);
}
- if (ST.hasPermLaneX16() && isFullRowPattern(Ids)) {
- auto [Lo, Hi] = computePermlane16Masks(Ids);
- return createPermlane16(B, Src, Lo, Hi);
+ if (ST.hasPermLaneX16()) {
+ if (isFullRowPattern(Ids)) {
+ auto [Lo, Hi] = computePermlane16Masks(Ids);
+ return createPermlane16(B, Src, Lo, Hi);
+ }
+ // Cross-row shuffles (e.g. XOR 16..31) — covered by permlanex16.
+ if (isCrossRowPattern(Ids)) {
+ auto [Lo, Hi] = computePermlane16Masks(Ids);
+ return createPermlaneX16(B, Src, Lo, Hi);
+ }
}
// DS_SWIZZLE bitmask-mode fallback for targets without DPP8/permlane16.
@@ -1846,12 +1823,8 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
return IC.replaceOperand(II, 0, PoisonValue::get(VDstIn->getType()));
}
- case Intrinsic::amdgcn_wave_shuffle: {
- if (ST->hasDPP())
- if (std::optional<Instruction *> R = tryWaveShuffleDPP(*ST, IC, II))
- return R;
+ case Intrinsic::amdgcn_wave_shuffle:
return tryOptimizeShufflePattern(IC, II, *ST);
- }
case Intrinsic::amdgcn_permlane64:
case Intrinsic::amdgcn_readfirstlane:
case Intrinsic::amdgcn_readlane:
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
index ed415313b941c..582789501dd41 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.wave.shuffle.ll
@@ -35,12 +35,12 @@ define i32 @test_wave_shuffle_self_select(i32 %val) {
define i32 @test_wave_shuffle_dpp_row_share_0(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT: ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 true)
; CHECK-W64-NEXT: ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0(
@@ -62,12 +62,12 @@ define i32 @test_wave_shuffle_dpp_row_share_0(i32 %val) {
define i32 @test_wave_shuffle_dpp_row_share_0_no_or(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT: ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 false)
+; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 336, i32 15, i32 15, i1 true)
; CHECK-W64-NEXT: ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_0_no_or(
@@ -88,12 +88,12 @@ define i32 @test_wave_shuffle_dpp_row_share_0_no_or(i32 %val) {
define i32 @test_wave_shuffle_dpp_row_share_7(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT: ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
; CHECK-W64-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; CHECK-W64-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 true)
; CHECK-W64-NEXT: ret i32 [[RES]]
;
; CHECK-NO-WAVE-SIZE-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7(
@@ -145,7 +145,7 @@ define i32 @test_wave_shuffle_dpp_row_share_7_no_mask(i32 %val) {
define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT: ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(
@@ -177,7 +177,7 @@ define i32 @test_wave_shuffle_dpp_row_share_7_lo_only(i32 %val) {
define i32 @test_wave_shuffle_dpp_row_share_w32_mask(i32 %val) {
; CHECK-W32-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
; CHECK-W32-SAME: i32 [[VAL:%.*]]) #[[ATTR0]] {
-; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 false)
+; CHECK-W32-NEXT: [[RES:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL]], i32 343, i32 15, i32 15, i1 true)
; CHECK-W32-NEXT: ret i32 [[RES]]
;
; CHECK-W64-LABEL: define i32 @test_wave_shuffle_dpp_row_share_w32_mask(
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
index fa75823f6c9f5..78b4948243715 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
@@ -149,7 +149,7 @@ define i32 @test_row_rotate_right3_w32(i32 %val) {
; Row share: broadcast lane 3 within each 16-lane row.
define i32 @test_row_share3_w32(i32 %val) {
; GFX11-LABEL: @test_row_share3_w32(
-; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 339, i32 15, i32 15, i1 false)
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 339, i32 15, i32 15, i1 true)
; GFX11-NEXT: ret i32 [[RESULT]]
;
; GFX11-W64-LABEL: @test_row_share3_w32(
@@ -294,6 +294,78 @@ define i32 @test_permlane16_mul3_w32(i32 %val) {
ret i32 %result
}
+; PermlaneX16: lane ^ 16. Cross-row swap; identity selectors.
+define i32 @test_permlanex16_xor16_w32(i32 %val) {
+; GFX11-LABEL: @test_permlanex16_xor16_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_permlanex16_xor16_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 16
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_permlanex16_xor16_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 16
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 16
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; PermlaneX16: lane ^ 17. Cross-row + XOR-by-1 within row.
+define i32 @test_permlanex16_xor17_w32(i32 %val) {
+; GFX11-LABEL: @test_permlanex16_xor17_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1732584193, i32 -271733879, i1 false, i1 false)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_permlanex16_xor17_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 17
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_permlanex16_xor17_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 17
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 17
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; PermlaneX16: lane ^ 31. Cross-row + reverse within row.
+define i32 @test_permlanex16_xor31_w32(i32 %val) {
+; GFX11-LABEL: @test_permlanex16_xor31_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 -1985229329, i32 19088743, i1 false, i1 false)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_permlanex16_xor31_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 31
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_permlanex16_xor31_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 31
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i32 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 31
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
; Float type should work the same as i32.
define float @test_quad_perm_xor1_float_w32(float %val) {
; GFX11-LABEL: @test_quad_perm_xor1_float_w32(
>From b6efc14a5f9e84a3f6f9c3af255b091e71711672 Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Thu, 30 Apr 2026 13:06:06 +0200
Subject: [PATCH 6/9] Added more tests
---
.../AMDGPU/wave-shuffle-patterns.ll | 76 +++++++++++++++++++
1 file changed, 76 insertions(+)
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
index 78b4948243715..c6960bc6eb377 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
@@ -7,7 +7,9 @@ declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
declare i32 @llvm.amdgcn.wave.shuffle.i32(i32, i32)
declare float @llvm.amdgcn.wave.shuffle.f32(float, i32)
+declare <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16>, i32)
declare ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3), i32)
+declare ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5), i32)
declare i32 @llvm.amdgcn.ds.bpermute(i32, i32)
; Quad perm: lane ^ 1 swaps adjacent pairs.
@@ -415,6 +417,80 @@ define ptr addrspace(3) @test_quad_perm_xor1_ptr3_w32(ptr addrspace(3) %val) {
ret ptr addrspace(3) %result
}
+; Scratch (32-bit) pointer: same gate must accept other 32-bit address spaces.
+define ptr addrspace(5) @test_quad_perm_xor1_ptr5_w32(ptr addrspace(5) %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_ptr5_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.update.dpp.p5(ptr addrspace(5) poison, ptr addrspace(5) [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret ptr addrspace(5) [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_ptr5_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret ptr addrspace(5) [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_ptr5_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT: [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret ptr addrspace(5) [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %result = call ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5) %val, i32 %idx)
+ ret ptr addrspace(5) %result
+}
+
+; <2 x i16> packed vector should optimize like i32 (same 32-bit width).
+define <2 x i16> @test_quad_perm_xor1_v2i16_w32(<2 x i16> %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_v2i16_w32(
+; GFX11-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.update.dpp.v2i16(<2 x i16> poison, <2 x i16> [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret <2 x i16> [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_v2i16_w32(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret <2 x i16> [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_v2i16_w32(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret <2 x i16> [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %result = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> %val, i32 %idx)
+ ret <2 x i16> %result
+}
+
+; Half-row-share with <2 x i16>: on GFX9 (no DPP8) falls into ds_swizzle,
+; exercising the bitcast path in createDsSwizzle. Uses the full
+; mbcnt.lo+mbcnt.hi sequence so isThreadID succeeds on wave64 (gfx900).
+define <2 x i16> @test_half_row_share3_v2i16(<2 x i16> %val) {
+; GFX11-LABEL: @test_half_row_share3_v2i16(
+; GFX11-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.mov.dpp8.v2i16(<2 x i16> [[VAL:%.*]], i32 7190235)
+; GFX11-NEXT: ret <2 x i16> [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_half_row_share3_v2i16(
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.mov.dpp8.v2i16(<2 x i16> [[VAL:%.*]], i32 7190235)
+; GFX11-W64-NEXT: ret <2 x i16> [[RESULT]]
+;
+; GFX9-LABEL: @test_half_row_share3_v2i16(
+; GFX9-NEXT: [[TMP1:%.*]] = bitcast <2 x i16> [[VAL:%.*]] to i32
+; GFX9-NEXT: [[TMP2:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[TMP1]], i32 120)
+; GFX9-NEXT: [[RESULT:%.*]] = bitcast i32 [[TMP2]] to <2 x i16>
+; GFX9-NEXT: ret <2 x i16> [[RESULT]]
+;
+ %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %tid = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %masked = and i32 %tid, -8
+ %idx = or i32 %masked, 3
+ %result = call <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16> %val, i32 %idx)
+ ret <2 x i16> %result
+}
+
; Negative: non-constant index, should not be optimized.
define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
; GFX11-LABEL: @test_nonconstant_shuffle_w32(
>From 9521ecff5ec4ab57afc37d39762a876f56a92473 Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Thu, 30 Apr 2026 13:59:10 +0200
Subject: [PATCH 7/9] Added 64-bit and 16-bit lit tests
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 9 +-
.../AMDGPU/wave-shuffle-patterns.ll | 89 +++++++++++++++++--
2 files changed, 91 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index be98fc7595146..3ccd1dd6e133c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -664,12 +664,17 @@ static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
if (Depth >= MaxAnalysisRecursionDepth)
return std::nullopt;
- if (isThreadID(ST, V))
- return Lane;
+ // Poison/undef in the index expression: the shuffle result is poison
+ // regardless of which lane we evaluate.
+ if (isa<PoisonValue>(V) || isa<UndefValue>(V))
+ return std::nullopt;
if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
return CI->getZExtValue();
+ if (isThreadID(ST, V))
+ return Lane;
+
const BinaryOperator *BO = dyn_cast<BinaryOperator>(V);
if (!BO)
return std::nullopt;
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
index c6960bc6eb377..047f6d8191a15 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
@@ -6,10 +6,13 @@
declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
declare i32 @llvm.amdgcn.wave.shuffle.i32(i32, i32)
+declare i64 @llvm.amdgcn.wave.shuffle.i64(i64, i32)
+declare i16 @llvm.amdgcn.wave.shuffle.i16(i16, i32)
declare float @llvm.amdgcn.wave.shuffle.f32(float, i32)
declare <2 x i16> @llvm.amdgcn.wave.shuffle.v2i16(<2 x i16>, i32)
declare ptr addrspace(3) @llvm.amdgcn.wave.shuffle.p3(ptr addrspace(3), i32)
declare ptr addrspace(5) @llvm.amdgcn.wave.shuffle.p5(ptr addrspace(5), i32)
+declare ptr @llvm.amdgcn.wave.shuffle.p0(ptr, i32)
declare i32 @llvm.amdgcn.ds.bpermute(i32, i32)
; Quad perm: lane ^ 1 swaps adjacent pairs.
@@ -417,7 +420,7 @@ define ptr addrspace(3) @test_quad_perm_xor1_ptr3_w32(ptr addrspace(3) %val) {
ret ptr addrspace(3) %result
}
-; Scratch (32-bit) pointer: same gate must accept other 32-bit address spaces.
+; Scratch (32-bit) pointer, should optimize like other 32-bit pointers.
define ptr addrspace(5) @test_quad_perm_xor1_ptr5_w32(ptr addrspace(5) %val) {
; GFX11-LABEL: @test_quad_perm_xor1_ptr5_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call ptr addrspace(5) @llvm.amdgcn.update.dpp.p5(ptr addrspace(5) poison, ptr addrspace(5) [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
@@ -441,7 +444,7 @@ define ptr addrspace(5) @test_quad_perm_xor1_ptr5_w32(ptr addrspace(5) %val) {
ret ptr addrspace(5) %result
}
-; <2 x i16> packed vector should optimize like i32 (same 32-bit width).
+; <2 x i16> vector (32-bit), should optimize like i32.
define <2 x i16> @test_quad_perm_xor1_v2i16_w32(<2 x i16> %val) {
; GFX11-LABEL: @test_quad_perm_xor1_v2i16_w32(
; GFX11-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.update.dpp.v2i16(<2 x i16> poison, <2 x i16> [[VAL:%.*]], i32 177, i32 15, i32 15, i1 true)
@@ -465,9 +468,7 @@ define <2 x i16> @test_quad_perm_xor1_v2i16_w32(<2 x i16> %val) {
ret <2 x i16> %result
}
-; Half-row-share with <2 x i16>: on GFX9 (no DPP8) falls into ds_swizzle,
-; exercising the bitcast path in createDsSwizzle. Uses the full
-; mbcnt.lo+mbcnt.hi sequence so isThreadID succeeds on wave64 (gfx900).
+; Half-row-share with <2 x i16>: falls back to ds_swizzle on GFX9, using the bitcast path.
define <2 x i16> @test_half_row_share3_v2i16(<2 x i16> %val) {
; GFX11-LABEL: @test_half_row_share3_v2i16(
; GFX11-NEXT: [[RESULT:%.*]] = call <2 x i16> @llvm.amdgcn.mov.dpp8.v2i16(<2 x i16> [[VAL:%.*]], i32 7190235)
@@ -491,6 +492,84 @@ define <2 x i16> @test_half_row_share3_v2i16(<2 x i16> %val) {
ret <2 x i16> %result
}
+; Negative: 64-bit type, should not optimize.
+define i64 @test_quad_perm_xor1_i64_negative(i64 %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_i64_negative(
+; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-NEXT: ret i64 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_i64_negative(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i64 [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_i64_negative(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT: [[RESULT:%.*]] = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i64 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %result = call i64 @llvm.amdgcn.wave.shuffle.i64(i64 %val, i32 %idx)
+ ret i64 %result
+}
+
+; Negative: 64-bit pointer, should not optimize.
+define ptr @test_quad_perm_xor1_p0_negative(ptr %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_p0_negative(
+; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-NEXT: [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
+; GFX11-NEXT: ret ptr [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_p0_negative(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret ptr [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_p0_negative(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT: [[RESULT:%.*]] = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret ptr [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %result = call ptr @llvm.amdgcn.wave.shuffle.p0(ptr %val, i32 %idx)
+ ret ptr %result
+}
+
+; Negative: 16-bit type, should not optimize.
+define i16 @test_quad_perm_xor1_i16_negative(i16 %val) {
+; GFX11-LABEL: @test_quad_perm_xor1_i16_negative(
+; GFX11-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-NEXT: [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-NEXT: ret i16 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_quad_perm_xor1_i16_negative(
+; GFX11-W64-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX11-W64-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
+; GFX11-W64-NEXT: ret i16 [[RESULT]]
+;
+; GFX9-LABEL: @test_quad_perm_xor1_i16_negative(
+; GFX9-NEXT: [[LANE:%.*]] = call range(i32 0, 33) i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+; GFX9-NEXT: [[IDX:%.*]] = xor i32 [[LANE]], 1
+; GFX9-NEXT: [[RESULT:%.*]] = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 [[VAL:%.*]], i32 [[IDX]])
+; GFX9-NEXT: ret i16 [[RESULT]]
+;
+ %lane = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %idx = xor i32 %lane, 1
+ %result = call i16 @llvm.amdgcn.wave.shuffle.i16(i16 %val, i32 %idx)
+ ret i16 %result
+}
+
; Negative: non-constant index, should not be optimized.
define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
; GFX11-LABEL: @test_nonconstant_shuffle_w32(
>From 5b835d40033b0426f82d24d2a82d1a0a215de306 Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Tue, 5 May 2026 13:28:54 +0200
Subject: [PATCH 8/9] Added poison test
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 7 ++++---
.../AMDGPU/wave-shuffle-patterns.ll | 18 ++++++++++++++++++
2 files changed, 22 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 3ccd1dd6e133c..7c3aad14a429e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -21,6 +21,7 @@
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
@@ -664,9 +665,9 @@ static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
if (Depth >= MaxAnalysisRecursionDepth)
return std::nullopt;
- // Poison/undef in the index expression: the shuffle result is poison
- // regardless of which lane we evaluate.
- if (isa<PoisonValue>(V) || isa<UndefValue>(V))
+ // Poison/undef in the index expression: bail and let InstCombine fold the
+ // intrinsic the usual way.
+ if (isa<UndefValue>(V))
return std::nullopt;
if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
index 047f6d8191a15..a167db139b244 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
@@ -570,6 +570,24 @@ define i16 @test_quad_perm_xor1_i16_negative(i16 %val) {
ret i16 %result
}
+; Negative: poison index, should not optimize.
+define i32 @test_poison_index_negative(i32 %val) {
+; GFX11-LABEL: @test_poison_index_negative(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_poison_index_negative(
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_poison_index_negative(
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 [[VAL:%.*]], i32 poison)
+; GFX9-NEXT: ret i32 [[RESULT]]
+;
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 poison)
+ ret i32 %result
+}
+
; Negative: non-constant index, should not be optimized.
define i32 @test_nonconstant_shuffle_w32(i32 %val, i32 %idx) {
; GFX11-LABEL: @test_nonconstant_shuffle_w32(
>From 0d5bf85199ab9eb84dc3141c244e4f285f614552 Mon Sep 17 00:00:00 2001
From: aleksandar <aleksandar.spasojevic at amd.com>
Date: Tue, 12 May 2026 13:36:15 +0200
Subject: [PATCH 9/9] DS_SWIZZLE matching correction
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 188 ++++++++++++------
.../AMDGPU/wave-shuffle-patterns.ll | 70 +++++++
2 files changed, 195 insertions(+), 63 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 7c3aad14a429e..542f17ff980b8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -19,11 +19,14 @@
#include "GCNSubtarget.h"
#include "SIDefines.h"
#include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
+#include "llvm/Support/MathExtras.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
#include <optional>
@@ -658,6 +661,8 @@ GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
return nullptr;
}
+/// Evaluate V as a function of the lane ID and return its value on Lane, or
+/// std::nullopt if V is not a closed-form expression of the lane ID.
static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
const GCNSubtarget &ST,
const DataLayout &DL,
@@ -698,35 +703,38 @@ static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
return CI->getZExtValue();
}
+/// Build the per-lane shuffle map by evaluating Index for every lane in the
+/// wave. Returns false if any lane index is non-constant or out of range.
static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
SmallVectorImpl<uint8_t> &Ids,
const DataLayout &DL) {
unsigned WaveSize = ST.getWavefrontSize();
Ids.resize(WaveSize);
- for (unsigned Lane = 0; Lane < WaveSize; ++Lane) {
- auto Val = evalLaneExpr(Index, Lane, ST, DL);
+ for (unsigned Lane : seq(WaveSize)) {
+ std::optional<unsigned> Val = evalLaneExpr(Index, Lane, ST, DL);
if (!Val || *Val >= WaveSize)
return false;
- Ids[Lane] = static_cast<uint8_t>(*Val);
+ Ids[Lane] = *Val;
}
return true;
}
/// Lanes are partitioned into groups of Period; each group is a translated
-/// copy of the first: Ids[i] = Ids[i % Period] + (i & ~(Period - 1)).
+/// copy of the first: Ids[I] = Ids[I % Period] + (I & ~(Period - 1)).
template <unsigned Period>
static bool hasPeriodicLayout(ArrayRef<uint8_t> Ids) {
- static_assert(Period > 0 && (Period & (Period - 1)) == 0,
- "Period must be a power of two");
- for (unsigned i = Period, E = Ids.size(); i < E; ++i)
- if (Ids[i] != Ids[i % Period] + (i & ~(Period - 1)))
+ static_assert(isPowerOf2_32(Period), "Period must be a power of two");
+ for (unsigned I = Period, E = Ids.size(); I < E; ++I)
+ if (Ids[I] != Ids[I % Period] + (I & ~(Period - 1)))
return false;
return true;
}
+/// Match an N-lane row pattern: each lane in [0, N) reads from a source lane
+/// in the same N-lane row, and the pattern repeats periodically across rows.
template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
- for (unsigned i = 0; i < N; ++i)
- if (Ids[i] >= N)
+ for (unsigned I = 0; I < N; ++I)
+ if (Ids[I] >= N)
return false;
return hasPeriodicLayout<N>(Ids);
}
@@ -735,17 +743,21 @@ static constexpr auto isQuadPattern = isRowPattern<4>;
static constexpr auto isHalfRowPattern = isRowPattern<8>;
static constexpr auto isFullRowPattern = isRowPattern<16>;
+/// Match a 4-lane (quad) permutation, encoded as the v_mov_b32_dpp
+/// QUAD_PERM control word: bits[1:0]=Ids[0], [3:2]=Ids[1], [5:4]=Ids[2],
+/// [7:6]=Ids[3].
static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
if (!isQuadPattern(Ids))
return std::nullopt;
- return (Ids[3] << 6) | (Ids[2] << 4) | (Ids[1] << 2) | Ids[0];
+ return Ids[3] << 6 | Ids[2] << 4 | Ids[1] << 2 | Ids[0];
}
+/// Match an N-lane reversal (mirror) pattern.
template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
if (!isRowPattern<N>(Ids))
return false;
- for (unsigned j = 0; j < N; ++j)
- if (Ids[j] != (N - 1) - j)
+ for (unsigned J = 0; J < N; ++J)
+ if (Ids[J] != (N - 1) - J)
return false;
return true;
}
@@ -753,92 +765,132 @@ template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
static constexpr auto matchHalfRowMirrorPattern = matchMirrorPattern<8>;
static constexpr auto matchFullRowMirrorPattern = matchMirrorPattern<16>;
+/// Match a 16-lane cyclic rotation; returns the rotation amount in [1, 15].
static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
if (!isFullRowPattern(Ids))
return std::nullopt;
- if (Ids[0] == 0 || Ids[15] == 15)
+ if (Ids[0] == 0)
return std::nullopt;
- for (unsigned j = 1; j < 16; ++j) {
- int Diff = static_cast<int>(Ids[j]) - static_cast<int>(Ids[j - 1]);
- if (Diff != 1 && Diff != -15)
+ for (unsigned J = 1; J < 16; ++J)
+ if (Ids[J] != (Ids[0] + J) % 16)
return std::nullopt;
- }
return 16u - Ids[0];
}
+/// Match a row-share pattern: all 16 lanes of each row read the same source
+/// lane. Returns the shared source lane index in [0, 16).
static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
if (!isFullRowPattern(Ids))
return std::nullopt;
- for (unsigned j = 1; j < 16; ++j)
- if (Ids[j] != Ids[0])
- return std::nullopt;
- return static_cast<unsigned>(Ids[0]);
+ if (!all_equal(Ids.take_front(16)))
+ return std::nullopt;
+ return Ids[0];
}
+/// Match an XOR mask pattern within each 16-lane row: Ids[J] == Mask ^ J,
+/// with Mask in [1, 15].
static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
if (!isFullRowPattern(Ids))
return std::nullopt;
unsigned Mask = Ids[0];
if (Mask == 0)
return std::nullopt;
- for (unsigned j = 0; j < 16; ++j)
- if (Ids[j] != (Mask ^ j))
+ for (unsigned J = 0; J < 16; ++J)
+ if (Ids[J] != (Mask ^ J))
return std::nullopt;
return Mask;
}
+/// Match an 8-lane arbitrary permutation, encoded as the v_mov_b32_dpp8
+/// 24-bit selector (three bits per output lane).
static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
if (!isHalfRowPattern(Ids))
return std::nullopt;
unsigned Selector = 0;
- for (unsigned j = 0; j < 8; ++j)
- Selector |= static_cast<unsigned>(Ids[j] & 0x7) << (j * 3);
+ for (unsigned J = 0; J < 8; ++J)
+ Selector |= Ids[J] << (J * 3);
return Selector;
}
-static std::pair<uint32_t, uint32_t>
-computePermlane16Masks(ArrayRef<uint8_t> Ids) {
- uint32_t Lo = 0, Hi = 0;
- for (unsigned j = 0; j < 8; ++j)
- Lo |= static_cast<uint32_t>(Ids[j] & 0xF) << (j * 4);
- for (unsigned j = 8; j < 16; ++j)
- Hi |= static_cast<uint32_t>(Ids[j] & 0xF) << ((j - 8) * 4);
- return {Lo, Hi};
-}
-
-static std::optional<unsigned> matchHalfRowSharePattern(ArrayRef<uint8_t> Ids) {
- if (!isHalfRowPattern(Ids))
- return std::nullopt;
- for (unsigned j = 1; j < 8; ++j)
- if (Ids[j] != Ids[0])
- return std::nullopt;
- // Bitmask-mode: AND=0x18 (keep group-select bits), OR=ids[0], XOR=0
- return (static_cast<unsigned>(Ids[0]) << AMDGPU::Swizzle::BITMASK_OR_SHIFT) |
- 0x18u;
+/// Pack a 16-lane permutation into a single 64-bit value: four bits per output
+/// lane, lane J in bits [J*4 + 3 : J*4]. The caller splits it into the low and
+/// high 32-bit selector operands of v_permlane16 / v_permlanex16.
+static uint64_t computePermlane16Masks(ArrayRef<uint8_t> Ids) {
+ uint64_t Sel = 0;
+ for (unsigned J = 0; J < 16; ++J)
+ Sel |= static_cast<uint64_t>(Ids[J] & 0xF) << (J * 4);
+ return Sel;
}
+/// Match a half-wave swap: lane J reads from lane J ^ 32. Only meaningful on
+/// wave64 targets.
static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
if (Ids.size() != 64)
return false;
- for (unsigned j = 0; j < 64; ++j)
- if (Ids[j] != (j ^ 32))
+ for (unsigned J = 0; J < 64; ++J)
+ if (Ids[J] != (J ^ 32))
return false;
return true;
}
-// Detects a cross-row shuffle pattern suitable for v_permlanex16
+/// Match a cross-row permutation suitable for v_permlanex16: every lane in
+/// the low 16-lane half reads from the high half of its own row, and vice
+/// versa.
static bool isCrossRowPattern(ArrayRef<uint8_t> Ids) {
- if (Ids.size() < 32 || !hasPeriodicLayout<32>(Ids))
+ if (!hasPeriodicLayout<32>(Ids))
return false;
- for (unsigned j = 0; j < 16; ++j) {
- if (Ids[j] < 16 || Ids[j] >= 32)
+ for (unsigned J = 0; J < 16; ++J) {
+ if (Ids[J] < 16 || Ids[J] >= 32)
return false;
- if (Ids[j + 16] != Ids[j] - 16)
+ if (Ids[J + 16] != Ids[J] - 16)
return false;
}
return true;
}
+/// Match a DS_SWIZZLE bitmask-mode permutation:
+/// dst_lane = ((src_lane & AND) | OR) ^ XOR
+/// with each mask being five bits. Returns the encoded swizzle immediate.
+/// The hardware applies the formula independently within each 32-lane group,
+/// so on wave64 the high group must replicate the low one (translated by 32).
+static std::optional<unsigned>
+matchDsSwizzleBitmaskPattern(ArrayRef<uint8_t> Ids) {
+ if (!hasPeriodicLayout<32>(Ids))
+ return std::nullopt;
+
+ // The formula is per-bit: output bit B depends only on input bit B. Probe
+ // each bit with src=0 and src=(1<<B); if the output bit flipped, AND[B]=1
+ // and XOR[B] carries the constant offset; otherwise it is a constant bit
+ // encoded in OR (with AND[B]=0, XOR[B]=0).
+ unsigned AndMask = 0, OrMask = 0, XorMask = 0;
+ for (unsigned B = 0; B < 5; ++B) {
+ unsigned Bit0 = (Ids[0] >> B) & 1;
+ unsigned Bit1 = (Ids[1u << B] >> B) & 1;
+ if (Bit0 != Bit1) {
+ AndMask |= 1u << B;
+ XorMask |= Bit0 << B;
+ } else {
+ OrMask |= Bit0 << B;
+ }
+ }
+
+ // The per-bit derivation assumes bit independence; verify the masks
+ // actually reproduce every lane in the 32-lane group.
+ for (unsigned I : seq(32u)) {
+ unsigned Expected = ((I & AndMask) | OrMask) ^ XorMask;
+ if (Ids[I] != Expected)
+ return std::nullopt;
+ }
+
+ return AMDGPU::Swizzle::BITMASK_PERM_ENC |
+ (AndMask << AMDGPU::Swizzle::BITMASK_AND_SHIFT) |
+ (OrMask << AMDGPU::Swizzle::BITMASK_OR_SHIFT) |
+ (XorMask << AMDGPU::Swizzle::BITMASK_XOR_SHIFT);
+}
+
+/// Emit v_mov_b32_dpp with the given control word, row/bank masks 0xF, and
+/// bound_ctrl=1 so out-of-bounds lanes are well-defined and the DPP mov can
+/// be folded into a consuming VALU op by GCNDPPCombine.
static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
Type *Ty = Val->getType();
return B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, {Ty},
@@ -846,11 +898,13 @@ static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
B.getInt32(0xF), B.getInt32(0xF), B.getTrue()});
}
+/// Emit v_mov_b32_dpp8 with the given 24-bit lane selector.
static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector) {
return B.CreateIntrinsic(Intrinsic::amdgcn_mov_dpp8, {Val->getType()},
{Val, B.getInt32(Selector)});
}
+/// Emit v_permlane16 with the precomputed lane-select halves.
static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
uint32_t Hi) {
Type *Ty = Val->getType();
@@ -859,6 +913,8 @@ static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
B.getInt32(Hi), B.getFalse(), B.getFalse()});
}
+/// Emit v_permlanex16 with the precomputed lane-select halves. Each output
+/// lane reads from the other 16-lane half of the same row.
static Value *createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo,
uint32_t Hi) {
Type *Ty = Val->getType();
@@ -867,6 +923,8 @@ static Value *createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo,
B.getInt32(Hi), B.getFalse(), B.getFalse()});
}
+/// Emit ds_swizzle with the given immediate, bitcasting/converting between
+/// pointer/float types and i32 as required by the intrinsic signature.
static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
const DataLayout &DL) {
Type *OrigTy = Val->getType();
@@ -876,17 +934,18 @@ static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
Value *Src = Val;
if (OrigTy->isPointerTy())
Src = B.CreatePtrToInt(Src, I32Ty);
- else if (!OrigTy->isIntegerTy(32))
+ else if (OrigTy != I32Ty)
Src = B.CreateBitCast(Src, I32Ty);
Value *Result = B.CreateIntrinsic(Intrinsic::amdgcn_ds_swizzle, {},
{Src, B.getInt32(Offset)});
if (OrigTy->isPointerTy())
return B.CreateIntToPtr(Result, OrigTy);
- if (!OrigTy->isIntegerTy(32))
+ if (OrigTy != I32Ty)
return B.CreateBitCast(Result, OrigTy);
return Result;
}
+/// Emit v_permlane64 (swap of the two 32-lane halves of a wave64).
static Value *createPermlane64(IRBuilderBase &B, Value *Val) {
return B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {Val->getType()},
{Val});
@@ -930,19 +989,22 @@ static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
if (ST.hasPermLaneX16()) {
if (isFullRowPattern(Ids)) {
- auto [Lo, Hi] = computePermlane16Masks(Ids);
- return createPermlane16(B, Src, Lo, Hi);
+ uint64_t Sel = computePermlane16Masks(Ids);
+ return createPermlane16(B, Src, Lo_32(Sel), Hi_32(Sel));
}
// Cross-row shuffles (e.g. XOR 16..31) — covered by permlanex16.
if (isCrossRowPattern(Ids)) {
- auto [Lo, Hi] = computePermlane16Masks(Ids);
- return createPermlaneX16(B, Src, Lo, Hi);
+ uint64_t Sel = computePermlane16Masks(Ids);
+ return createPermlaneX16(B, Src, Lo_32(Sel), Hi_32(Sel));
}
}
- // DS_SWIZZLE bitmask-mode fallback for targets without DPP8/permlane16.
- if (std::optional<unsigned> Offset = matchHalfRowSharePattern(Ids))
- return createDsSwizzle(B, Src, *Offset, DL);
+ // Generic DS_SWIZZLE bitmask-mode fallback: handles any 32-lane shuffle that
+ // can be expressed as dst = ((src & AND) | OR) ^ XOR with 5-bit masks. This
+ // is available on every target that has ds_swizzle (i.e. all of them) and
+ // strictly subsumes the half-row "share" cases the original code handled.
+ if (std::optional<unsigned> Imm = matchDsSwizzleBitmaskPattern(Ids))
+ return createDsSwizzle(B, Src, *Imm, DL);
if (ST.hasPermLane64() && matchHalfWaveSwapPattern(Ids))
return createPermlane64(B, Src);
@@ -970,11 +1032,11 @@ tryOptimizeShufflePattern(InstCombiner &IC, IntrinsicInst &II,
SmallVector<uint8_t, 64> Ids;
if (IsBpermute) {
Ids.resize(WaveSize);
- for (unsigned Lane = 0; Lane < WaveSize; ++Lane) {
+ for (unsigned Lane : seq(WaveSize)) {
std::optional<unsigned> Val = evalLaneExpr(Index, Lane, ST, DL);
if (!Val || (*Val & 3) || (*Val >> 2) >= WaveSize)
return std::nullopt;
- Ids[Lane] = static_cast<uint8_t>(*Val >> 2);
+ Ids[Lane] = *Val >> 2;
}
} else {
if (!tryBuildShuffleMap(Index, ST, Ids, DL))
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
index a167db139b244..290c0f39f0b76 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/wave-shuffle-patterns.ll
@@ -679,3 +679,73 @@ define i32 @test_bpermute_quad_xor1_w32(i32 %val) {
%result = call i32 @llvm.amdgcn.ds.bpermute(i32 %addr, i32 %val)
ret i32 %result
}
+
+; XOR-by-5 within each 16-lane row. On GFX10+ this matches DPP row_xmask;
+; on GFX9 row_xmask is unavailable, so the generic ds_swizzle bitmask path
+; takes over: AND=0x1F, OR=0, XOR=5 -> 5151.
+define i32 @test_xor5_bitmask(i32 %val) {
+; GFX11-LABEL: @test_xor5_bitmask(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_xor5_bitmask(
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 357, i32 15, i32 15, i1 true)
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_xor5_bitmask(
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 5151)
+; GFX9-NEXT: ret i32 [[RESULT]]
+;
+ %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %idx = xor i32 %lane, 5
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; XOR-by-16 (cross-row). On GFX11 this matches permlanex16; on GFX9 neither
+; permlanex16 nor row_xmask exist, so the bitmask path catches it:
+; AND=0x1F, OR=0, XOR=16 -> 16415.
+define i32 @test_xor16_bitmask(i32 %val) {
+; GFX11-LABEL: @test_xor16_bitmask(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_xor16_bitmask(
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.permlanex16.i32(i32 poison, i32 [[VAL:%.*]], i32 1985229328, i32 -19088744, i1 false, i1 false)
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_xor16_bitmask(
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 16415)
+; GFX9-NEXT: ret i32 [[RESULT]]
+;
+ %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %idx = xor i32 %lane, 16
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
+
+; Broadcast lane 5 within each 16-lane row (lanes 0..15 read lane 5, lanes
+; 16..31 read lane 21, etc.). On GFX10+ row_share covers it; on GFX9 there
+; is no row_share, so the bitmask path emits AND=0x10, OR=5, XOR=0 -> 176.
+define i32 @test_broadcast_in_rows16_bitmask(i32 %val) {
+; GFX11-LABEL: @test_broadcast_in_rows16_bitmask(
+; GFX11-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 341, i32 15, i32 15, i1 true)
+; GFX11-NEXT: ret i32 [[RESULT]]
+;
+; GFX11-W64-LABEL: @test_broadcast_in_rows16_bitmask(
+; GFX11-W64-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.update.dpp.i32(i32 poison, i32 [[VAL:%.*]], i32 341, i32 15, i32 15, i1 true)
+; GFX11-W64-NEXT: ret i32 [[RESULT]]
+;
+; GFX9-LABEL: @test_broadcast_in_rows16_bitmask(
+; GFX9-NEXT: [[RESULT:%.*]] = call i32 @llvm.amdgcn.ds.swizzle(i32 [[VAL:%.*]], i32 176)
+; GFX9-NEXT: ret i32 [[RESULT]]
+;
+ %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+ %lane = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+ %hi = and i32 %lane, -16
+ %idx = or i32 %hi, 5
+ %result = call i32 @llvm.amdgcn.wave.shuffle.i32(i32 %val, i32 %idx)
+ ret i32 %result
+}
More information about the llvm-commits
mailing list