[llvm] [AMDGPU][InstCombine] Optimize constant shuffle patterns (PR #192246)
Pankaj Dwivedi via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 29 06:29:09 PDT 2026
================
@@ -718,6 +657,331 @@ GCNTTIImpl::hoistLaneIntrinsicThroughOperand(InstCombiner &IC,
return nullptr;
}
+static std::optional<unsigned> evalLaneExpr(Value *V, unsigned Lane,
+ const GCNSubtarget &ST,
+ const DataLayout &DL,
+ unsigned Depth = 0) {
+ if (Depth >= MaxAnalysisRecursionDepth)
+ return std::nullopt;
+
+ if (isThreadID(ST, V))
+ return Lane;
+
+ if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
+ return CI->getZExtValue();
+
+ const BinaryOperator *BO = dyn_cast<BinaryOperator>(V);
+ if (!BO)
+ return std::nullopt;
+
+ std::optional<unsigned> LHS =
+ evalLaneExpr(BO->getOperand(0), Lane, ST, DL, Depth + 1);
+ if (!LHS)
+ return std::nullopt;
+ std::optional<unsigned> RHS =
+ evalLaneExpr(BO->getOperand(1), Lane, ST, DL, Depth + 1);
+ if (!RHS)
+ return std::nullopt;
+
+ Type *Ty = BO->getType();
+ Constant *Ops[] = {ConstantInt::get(Ty, *LHS), ConstantInt::get(Ty, *RHS)};
+ auto *CI =
+ dyn_cast_or_null<ConstantInt>(ConstantFoldInstOperands(BO, Ops, DL));
+ if (!CI)
+ return std::nullopt;
+ return CI->getZExtValue();
+}
+
+static bool tryBuildShuffleMap(Value *Index, const GCNSubtarget &ST,
+ SmallVectorImpl<uint8_t> &Ids,
+ const DataLayout &DL) {
+ unsigned WaveSize = ST.getWavefrontSize();
+ Ids.resize(WaveSize);
+ for (unsigned Lane = 0; Lane < WaveSize; ++Lane) {
+ auto Val = evalLaneExpr(Index, Lane, ST, DL);
+ if (!Val || *Val >= WaveSize)
+ return false;
+ Ids[Lane] = static_cast<uint8_t>(*Val);
+ }
+ return true;
+}
+
+/// Lanes are partitioned into groups of Period; each group is a translated
+/// copy of the first: Ids[i] = Ids[i % Period] + (i & ~(Period - 1)).
+template <unsigned Period>
+static bool hasPeriodicLayout(ArrayRef<uint8_t> Ids) {
+ static_assert(Period > 0 && (Period & (Period - 1)) == 0,
+ "Period must be a power of two");
+ for (unsigned i = Period, E = Ids.size(); i < E; ++i)
+ if (Ids[i] != Ids[i % Period] + (i & ~(Period - 1)))
+ return false;
+ return true;
+}
+
+template <unsigned N> static bool isRowPattern(ArrayRef<uint8_t> Ids) {
+ for (unsigned i = 0; i < N; ++i)
+ if (Ids[i] >= N)
+ return false;
+ return hasPeriodicLayout<N>(Ids);
+}
+
+static constexpr auto isQuadPattern = isRowPattern<4>;
+static constexpr auto isHalfRowPattern = isRowPattern<8>;
+static constexpr auto isFullRowPattern = isRowPattern<16>;
+
+static std::optional<unsigned> matchQuadPermPattern(ArrayRef<uint8_t> Ids) {
+ if (!isQuadPattern(Ids))
+ return std::nullopt;
+ return (Ids[3] << 6) | (Ids[2] << 4) | (Ids[1] << 2) | Ids[0];
+}
+
+template <unsigned N> static bool matchMirrorPattern(ArrayRef<uint8_t> Ids) {
+ if (!isRowPattern<N>(Ids))
+ return false;
+ for (unsigned j = 0; j < N; ++j)
+ if (Ids[j] != (N - 1) - j)
+ return false;
+ return true;
+}
+
+static constexpr auto matchHalfRowMirrorPattern = matchMirrorPattern<8>;
+static constexpr auto matchFullRowMirrorPattern = matchMirrorPattern<16>;
+
+static std::optional<unsigned> matchRowRotatePattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return std::nullopt;
+ if (Ids[0] == 0 || Ids[15] == 15)
+ return std::nullopt;
+ for (unsigned j = 1; j < 16; ++j) {
+ int Diff = static_cast<int>(Ids[j]) - static_cast<int>(Ids[j - 1]);
+ if (Diff != 1 && Diff != -15)
+ return std::nullopt;
+ }
+ return 16u - Ids[0];
+}
+
+static std::optional<unsigned> matchRowSharePattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return std::nullopt;
+ for (unsigned j = 1; j < 16; ++j)
+ if (Ids[j] != Ids[0])
+ return std::nullopt;
+ return static_cast<unsigned>(Ids[0]);
+}
+
+static std::optional<unsigned> matchRowXMaskPattern(ArrayRef<uint8_t> Ids) {
+ if (!isFullRowPattern(Ids))
+ return std::nullopt;
+ unsigned Mask = Ids[0];
+ if (Mask == 0)
+ return std::nullopt;
+ for (unsigned j = 0; j < 16; ++j)
+ if (Ids[j] != (Mask ^ j))
+ return std::nullopt;
+ return Mask;
+}
+
+static std::optional<unsigned> matchHalfRowPermPattern(ArrayRef<uint8_t> Ids) {
+ if (!isHalfRowPattern(Ids))
+ return std::nullopt;
+ unsigned Selector = 0;
+ for (unsigned j = 0; j < 8; ++j)
+ Selector |= static_cast<unsigned>(Ids[j] & 0x7) << (j * 3);
+ return Selector;
+}
+
+static std::pair<uint32_t, uint32_t>
+computePermlane16Masks(ArrayRef<uint8_t> Ids) {
+ uint32_t Lo = 0, Hi = 0;
+ for (unsigned j = 0; j < 8; ++j)
+ Lo |= static_cast<uint32_t>(Ids[j] & 0xF) << (j * 4);
+ for (unsigned j = 8; j < 16; ++j)
+ Hi |= static_cast<uint32_t>(Ids[j] & 0xF) << ((j - 8) * 4);
+ return {Lo, Hi};
+}
+
+static std::optional<unsigned> matchHalfRowSharePattern(ArrayRef<uint8_t> Ids) {
+ if (!isHalfRowPattern(Ids))
+ return std::nullopt;
+ for (unsigned j = 1; j < 8; ++j)
+ if (Ids[j] != Ids[0])
+ return std::nullopt;
+ // Bitmask-mode: AND=0x18 (keep group-select bits), OR=ids[0], XOR=0
+ return (static_cast<unsigned>(Ids[0]) << AMDGPU::Swizzle::BITMASK_OR_SHIFT) |
+ 0x18u;
+}
+
+static bool matchHalfWaveSwapPattern(ArrayRef<uint8_t> Ids) {
+ if (Ids.size() != 64)
+ return false;
+ for (unsigned j = 0; j < 64; ++j)
+ if (Ids[j] != (j ^ 32))
+ return false;
+ return true;
+}
+
+// Detects a cross-row shuffle pattern suitable for v_permlanex16
+static bool isCrossRowPattern(ArrayRef<uint8_t> Ids) {
+ if (Ids.size() < 32 || !hasPeriodicLayout<32>(Ids))
+ return false;
+ for (unsigned j = 0; j < 16; ++j) {
+ if (Ids[j] < 16 || Ids[j] >= 32)
+ return false;
+ if (Ids[j + 16] != Ids[j] - 16)
+ return false;
+ }
+ return true;
+}
+
+static Value *createUpdateDpp(IRBuilderBase &B, Value *Val, unsigned Ctrl) {
+ Type *Ty = Val->getType();
+ return B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, {Ty},
+ {PoisonValue::get(Ty), Val, B.getInt32(Ctrl),
+ B.getInt32(0xF), B.getInt32(0xF), B.getTrue()});
+}
+
+static Value *createMovDpp8(IRBuilderBase &B, Value *Val, unsigned Selector) {
+ return B.CreateIntrinsic(Intrinsic::amdgcn_mov_dpp8, {Val->getType()},
+ {Val, B.getInt32(Selector)});
+}
+
+static Value *createPermlane16(IRBuilderBase &B, Value *Val, uint32_t Lo,
+ uint32_t Hi) {
+ Type *Ty = Val->getType();
+ return B.CreateIntrinsic(Intrinsic::amdgcn_permlane16, {Ty},
+ {PoisonValue::get(Ty), Val, B.getInt32(Lo),
+ B.getInt32(Hi), B.getFalse(), B.getFalse()});
+}
+
+static Value *createPermlaneX16(IRBuilderBase &B, Value *Val, uint32_t Lo,
+ uint32_t Hi) {
+ Type *Ty = Val->getType();
+ return B.CreateIntrinsic(Intrinsic::amdgcn_permlanex16, {Ty},
+ {PoisonValue::get(Ty), Val, B.getInt32(Lo),
+ B.getInt32(Hi), B.getFalse(), B.getFalse()});
+}
+
+static Value *createDsSwizzle(IRBuilderBase &B, Value *Val, unsigned Offset,
+ const DataLayout &DL) {
+ Type *OrigTy = Val->getType();
+ assert(DL.getTypeSizeInBits(OrigTy) == 32 &&
+ "ds_swizzle only supports 32-bit operands");
+ IntegerType *I32Ty = B.getInt32Ty();
+ Value *Src = Val;
+ if (OrigTy->isPointerTy())
+ Src = B.CreatePtrToInt(Src, I32Ty);
+ else if (!OrigTy->isIntegerTy(32))
+ Src = B.CreateBitCast(Src, I32Ty);
+ Value *Result = B.CreateIntrinsic(Intrinsic::amdgcn_ds_swizzle, {},
+ {Src, B.getInt32(Offset)});
+ if (OrigTy->isPointerTy())
+ return B.CreateIntToPtr(Result, OrigTy);
+ if (!OrigTy->isIntegerTy(32))
+ return B.CreateBitCast(Result, OrigTy);
+ return Result;
+}
+
+static Value *createPermlane64(IRBuilderBase &B, Value *Val) {
+ return B.CreateIntrinsic(Intrinsic::amdgcn_permlane64, {Val->getType()},
+ {Val});
+}
+
+/// Given a shuffle map, try to emit the best hardware intrinsic.
+static Value *matchShuffleToHWIntrinsic(IRBuilderBase &B, Value *Src,
+ ArrayRef<uint8_t> Ids,
+ const GCNSubtarget &ST,
+ const DataLayout &DL) {
+ if (std::optional<unsigned> QP = matchQuadPermPattern(Ids)) {
+ if (ST.hasDPP())
+ return createUpdateDpp(B, Src, *QP);
+ return createDsSwizzle(B, Src, AMDGPU::Swizzle::QUAD_PERM_ENC | *QP, DL);
+ }
+
+ if (ST.hasDPP()) {
+ if (matchHalfRowMirrorPattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_HALF_MIRROR);
+ if (matchFullRowMirrorPattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_MIRROR);
+ if (std::optional<unsigned> Amt = matchRowRotatePattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_ROR_FIRST + *Amt - 1);
+ }
+
+ // row_share is supported on GFX90A and GFX10+; row_xmask is GFX10+ only.
+ if (ST.hasDPPRowShare()) {
+ if (std::optional<unsigned> Lane = matchRowSharePattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_SHARE_FIRST + *Lane);
+ }
+
+ if (ST.hasDPP() && ST.hasGFX10Insts()) {
+ if (std::optional<unsigned> Mask = matchRowXMaskPattern(Ids))
+ return createUpdateDpp(B, Src, AMDGPU::DPP::ROW_XMASK_FIRST + *Mask);
+ }
+
+ if (ST.hasDPP8()) {
+ if (std::optional<unsigned> Sel = matchHalfRowPermPattern(Ids))
+ return createMovDpp8(B, Src, *Sel);
+ }
+
+ if (ST.hasPermLaneX16()) {
+ if (isFullRowPattern(Ids)) {
+ auto [Lo, Hi] = computePermlane16Masks(Ids);
+ return createPermlane16(B, Src, Lo, Hi);
+ }
+ // Cross-row shuffles (e.g. XOR 16..31) — covered by permlanex16.
+ if (isCrossRowPattern(Ids)) {
+ auto [Lo, Hi] = computePermlane16Masks(Ids);
+ return createPermlaneX16(B, Src, Lo, Hi);
----------------
PankajDwivedi-25 wrote:
For XOR masks 1-15, the current priority tries row_xmask (DPP) before permlane16. This makes sense because GCNDPPCombine can fold the DPP mov into a consuming VALU op, which isn't possible with permlane16?
However, Jay mentioned on
#194565 that if there's no consuming math op, permlane16 might be preferable since a bare v_mov_b32_dpp doesn't gain anything over permlane16.
However, at InstCombine time, we don't know whether a consuming op exists, so emitting DPP seems like the right default (no-lose: folding wins when possible, same cost when not).
Is this the intended reasoning for the priority order? Or is there a scenario where permlane16 should be preferred over DPP row_xmask?
https://github.com/llvm/llvm-project/pull/192246
More information about the llvm-commits
mailing list