[llvm-branch-commits] [llvm] [SLP] Cancel the phantom load saving on widened loads (PR #228904)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Oct 4 08:13:41 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Dmitry Sidorov (MrSidims)
<details>
<summary>Changes</summary>
On a target whose consecutive scalar loads coalesce, cancel the load
saving of a bundle feeding a cast that widens its lanes, in store
chains, value lists and integer reductions. Without it a load of
<4 x i8> through sitofp into a store chain vectorizes into longer code.
A bundle keeps its saving when the loaded or the widened lanes pack two
to a register, since the scalar form unpacks every lane. Floating point
reductions keep their own rule.
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
---
Patch is 62.63 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/228904.diff
5 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+99-14)
- (modified) llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll (+48-175)
- (modified) llvm/test/CodeGen/AMDGPU/slp-int-to-fp.ll (+10-10)
- (modified) llvm/test/Transforms/SLPVectorizer/AMDGPU/elementwise-coalesced-loads.ll (+153-24)
- (modified) llvm/test/Transforms/SLPVectorizer/AMDGPU/int-reduction-coalesced-loads.ll (+160-36)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 26a6119308360..3c5e2df4a6ecb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -481,6 +481,10 @@ class slpvectorizer::BoUpSLP {
Keep,
/// Every clean bundle loses its saving.
Cancel,
+ /// Only the clean bundles feeding a cast whose lanes get wider than the
+ /// loaded type before they leave the tree lose their saving, unless the
+ /// loaded or the widened lanes are of a packed type.
+ CancelWidened,
};
/// Calculates the cost of the subtrees, trims non-profitable ones and returns
@@ -2491,14 +2495,29 @@ class slpvectorizer::BoUpSLP {
const SmallDenseSet<unsigned, 8> &NodesToKeepBWs, unsigned &MaxDepthLevel,
bool &IsProfitableToDemote, bool IsTruncRoot) const;
+ /// \returns the widest type the lanes of \p TE reach before they leave the
+ /// tree through a reduction, an external use or a user that is neither a
+ /// cast nor narrowed. With \p ApplyMinBWs the lanes are first narrowed to
+ /// their minimum bit widths, as the vector code is.
+ /// \p HasExternalUses tells whether a node has scalars used outside of the
+ /// tree.
+ Type *getWidestLaneType(
+ const TreeEntry &TE, bool ApplyMinBWs,
+ function_ref<bool(const TreeEntry &)> HasExternalUses) const;
+
+ /// \returns true if two lanes of \p Ty fit a scalar register as a legal
+ /// vector type, so the target has packed arithmetic on them.
+ bool isPackedLaneType(Type *Ty) const;
+
/// \returns the load saving credited to the vectorized load bundle \p TE
/// that never materializes on a target whose consecutive scalar loads
/// coalesce into the same wide access, or zero if \p LoadSavings keeps it.
/// Only clean bundles with no reordering, reuse or bit-width reduction lose
- /// the saving.
- InstructionCost
- getCoalescedLoadPhantomSaving(const TreeEntry &TE,
- CoalescedLoadSavings LoadSavings) const;
+ /// the saving. \p HasExternalUses tells whether a node has scalars used
+ /// outside of the tree.
+ InstructionCost getCoalescedLoadPhantomSaving(
+ const TreeEntry &TE, CoalescedLoadSavings LoadSavings,
+ function_ref<bool(const TreeEntry &)> HasExternalUses) const;
/// Builds the list of reorderable operands on the edges \p Edges of the \p
/// UserTE, which allow reordering (i.e. the operands can be reordered because
@@ -13431,12 +13450,16 @@ InstructionCost BoUpSLP::getUnfusedFMulsPenalty(const TreeEntry &TE) const {
/// \returns which load bundles of a reduction over \p VL lose the saving they
/// are credited for although their scalar loads coalesce anyway. A reduction
-/// that would lose an fma has a stake in every load bundle, and any other
-/// reduction keeps every saving. A reassociable reduction loses no fma because
-/// its vector fmuls fuse into the reduction.
+/// that would lose an fma has a stake in every load bundle, an integer
+/// reduction only in the bundles whose lanes get widened, the same as an
+/// elementwise tree, and any other reduction keeps every saving. A
+/// reassociable reduction loses no fma because its vector fmuls fuse into the
+/// reduction.
static BoUpSLP::CoalescedLoadSavings
getReductionCoalescedLoadSavings(RecurKind RdxKind, FastMathFlags RdxFMF,
ArrayRef<Value *> VL) {
+ if (RecurrenceDescriptor::isIntegerRecurrenceKind(RdxKind))
+ return BoUpSLP::CoalescedLoadSavings::CancelWidened;
if (RdxKind != RecurKind::FAdd || !RdxFMF.allowContract() ||
RdxFMF.allowReassoc() || none_of(VL, [](Value *V) {
return match(V,
@@ -18594,9 +18617,51 @@ bool BoUpSLP::isTreeNotExtendable() const {
return Res;
}
-InstructionCost
-BoUpSLP::getCoalescedLoadPhantomSaving(const TreeEntry &TE,
- CoalescedLoadSavings LoadSavings) const {
+Type *BoUpSLP::getWidestLaneType(
+ const TreeEntry &TE, bool ApplyMinBWs,
+ function_ref<bool(const TreeEntry &)> HasExternalUses) const {
+ Type *WidestTy = nullptr;
+ auto Widen = [&](Type *Ty) {
+ if (!WidestTy ||
+ DL->getTypeSizeInBits(Ty) > DL->getTypeSizeInBits(WidestTy))
+ WidestTy = Ty;
+ };
+ for (const TreeEntry *E = &TE;;) {
+ Type *OrigTy = E->Scalars.front()->getType()->getScalarType();
+ Type *Ty = OrigTy;
+ if (auto It = MinBWs.find(E);
+ ApplyMinBWs && It != MinBWs.end() &&
+ It->second.first < DL->getTypeSizeInBits(Ty) && !HasExternalUses(*E))
+ Ty = IntegerType::get(Ty->getContext(), It->second.first);
+ Widen(Ty);
+ const TreeEntry *UserTE = E->UserTreeIndex.UserTE;
+ if (!UserTE) {
+ if (ApplyMinBWs && UserIgnoreList && E == &getRootNode())
+ Widen(getReductionType()->getScalarType());
+ else
+ Widen(OrigTy);
+ return WidestTy;
+ }
+ if (!UserTE->hasState() || UserTE->isGather() ||
+ (!Instruction::isCast(UserTE->getOpcode()) &&
+ !MinBWs.contains(UserTE))) {
+ Widen(OrigTy);
+ return WidestTy;
+ }
+ E = UserTE;
+ }
+}
+
+bool BoUpSLP::isPackedLaneType(Type *Ty) const {
+ unsigned Bits = DL->getTypeSizeInBits(Ty);
+ unsigned RegBits =
+ TTI->getRegisterBitWidth(TargetTransformInfo::RGK_Scalar).getFixedValue();
+ return 2 * Bits <= RegBits && TTI->isTypeLegal(FixedVectorType::get(Ty, 2));
+}
+
+InstructionCost BoUpSLP::getCoalescedLoadPhantomSaving(
+ const TreeEntry &TE, CoalescedLoadSavings LoadSavings,
+ function_ref<bool(const TreeEntry &)> HasExternalUses) const {
if (LoadSavings == CoalescedLoadSavings::Keep)
return 0;
if (!TE.hasState() || TE.isGather() || TE.getOpcode() != Instruction::Load ||
@@ -18616,6 +18681,20 @@ BoUpSLP::getCoalescedLoadPhantomSaving(const TreeEntry &TE,
if (!TTI->consecutiveLoadsCoalesce(LI0->getType(), TE.Scalars.size(),
BestAlign, LI0->getPointerAddressSpace()))
return 0;
+ if (LoadSavings == CoalescedLoadSavings::CancelWidened) {
+ const TreeEntry *UserTE = TE.UserTreeIndex.UserTE;
+ if (!UserTE || UserTE->isGather() || !UserTE->hasState() ||
+ !Instruction::isCast(UserTE->getOpcode()))
+ return 0;
+ Type *LoadedTy = LI0->getType()->getScalarType();
+ Type *WidestTy =
+ getWidestLaneType(*UserTE, /*ApplyMinBWs=*/true, HasExternalUses);
+ if (DL->getTypeSizeInBits(WidestTy) <= DL->getTypeSizeInBits(LoadedTy) ||
+ isPackedLaneType(LoadedTy) ||
+ isPackedLaneType(
+ getWidestLaneType(*UserTE, /*ApplyMinBWs=*/false, HasExternalUses)))
+ return 0;
+ }
InstructionCost ScalarLdCost = 0;
for (Value *V : TE.Scalars) {
auto *LI = cast<LoadInst>(V);
@@ -19486,6 +19565,12 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
return false;
return IsExternallyUsedV(V);
};
+ auto HasExternalUses = [&](const TreeEntry &TE) {
+ return any_of(TE.Scalars, [&](Value *V) {
+ return !(TE.hasCopyableElements() && TE.isCopyableElement(V)) &&
+ IsExternallyUsedV(V);
+ });
+ };
InstructionCost Cost = 0;
SmallDenseMap<const TreeEntry *, uint64_t> EntryToScale;
uint64_t PrevScale = 0;
@@ -19523,7 +19608,7 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
"Expected gather nodes with users only.");
InstructionCost C = getEntryCost(&TE, VectorizedVals, CheckedExtracts);
- C += getCoalescedLoadPhantomSaving(TE, LoadSavings);
+ C += getCoalescedLoadPhantomSaving(TE, LoadSavings, HasExternalUses);
uint64_t Scale = 0;
bool CostIsFree = C == 0;
// For gather/buildvector (and split-vectorize) entries, prefer the
@@ -19726,7 +19811,7 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
};
auto RecostEntry = [&](const TreeEntry *TE) {
InstructionCost C = getEntryCost(TE, VectorizedVals, CheckedExtracts);
- C += getCoalescedLoadPhantomSaving(*TE, LoadSavings);
+ C += getCoalescedLoadPhantomSaving(*TE, LoadSavings, HasExternalUses);
if (!C.isValid() || C == 0)
return C;
uint64_t Scale = EntryToScale.lookup(TE);
@@ -29719,7 +29804,7 @@ SLPVectorizerPass::vectorizeStoreChainImpl(ArrayRef<Value *> Chain, BoUpSLP &R,
InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable(
/*VectorizedVals=*/{}, /*RdxRoot=*/nullptr,
- BoUpSLP::CoalescedLoadSavings::Keep);
+ BoUpSLP::CoalescedLoadSavings::CancelWidened);
R.buildExternalUses();
Size = R.getCanonicalGraphSize() - R.getNumSplatSubtreeEntries();
@@ -30803,7 +30888,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
R.computeMinimumValueSizes();
InstructionCost TreeCost = R.calculateTreeCostAndTrimNonProfitable(
/*VectorizedVals=*/{}, /*RdxRoot=*/nullptr,
- BoUpSLP::CoalescedLoadSavings::Keep);
+ BoUpSLP::CoalescedLoadSavings::CancelWidened);
R.buildExternalUses();
InstructionCost Cost = R.getTreeCost(TreeCost);
diff --git a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
index 281221e8181fa..0385e52fbdc32 100644
--- a/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
+++ b/llvm/test/CodeGen/AMDGPU/slp-coalesced-loads.ll
@@ -327,11 +327,10 @@ define void @store4_sitofp(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v0, v[0:1], off
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_ashrrev_i16_e32 v1, 8, v0
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
; GFX90A-NEXT: v_cvt_f32_i32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
-; GFX90A-NEXT: v_cvt_f32_i32_sdwa v5, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v5, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v6, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2
+; GFX90A-NEXT: v_cvt_f32_i32_sdwa v7, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3
; GFX90A-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -392,10 +391,12 @@ define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) {
; GFX90A-LABEL: zext4_bv_loop:
; GFX90A: ; %bb.0: ; %entry
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 1, v0
+; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB4_1: ; %loop
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: global_load_dword v6, v[0:1], off
+; GFX90A-NEXT: global_load_dword v6, v[0:1], off offset:-1
; GFX90A-NEXT: s_add_u32 s6, s6, 1
; GFX90A-NEXT: v_add_co_u32_e32 v0, vcc, 4, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
@@ -417,12 +418,14 @@ define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) {
; GFX1030-LABEL: zext4_bv_loop:
; GFX1030: ; %bb.0: ; %entry
; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1
+; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1030-NEXT: v_mov_b32_e32 v4, 0xffff
; GFX1030-NEXT: s_mov_b64 s[6:7], 0
; GFX1030-NEXT: .p2align 6
; GFX1030-NEXT: .LBB4_1: ; %loop
; GFX1030-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1030-NEXT: global_load_dword v7, v[0:1], off
+; GFX1030-NEXT: global_load_dword v7, v[0:1], off offset:-1
; GFX1030-NEXT: s_add_u32 s6, s6, 1
; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, 4
; GFX1030-NEXT: s_addc_u32 s7, s7, 0
@@ -444,10 +447,11 @@ define void @zext4_bv_loop(ptr addrspace(1) %a, ptr addrspace(1) %out) {
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 1, v[0:1]
; GFX1250-NEXT: s_mov_b64 s[0:1], 0
; GFX1250-NEXT: .LBB4_1: ; %loop
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: global_load_b32 v6, v[0:1], off
+; GFX1250-NEXT: global_load_b32 v6, v[0:1], off offset:-1
; GFX1250-NEXT: s_add_nc_u64 s[0:1], s[0:1], 1
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], 4, v[0:1]
@@ -501,51 +505,21 @@ define i32 @dot8_sext_i8(ptr addrspace(1) %a, ptr addrspace(1) %b) {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[2:3], off
-; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshrrev_b16_e32 v0, 8, v4
-; GFX90A-NEXT: v_lshrrev_b16_e32 v1, 8, v5
+; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b16_e32 v2, 8, v6
-; GFX90A-NEXT: v_lshrrev_b16_e32 v3, 8, v7
-; GFX90A-NEXT: v_mul_i32_i24_sdwa v8, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX90A-NEXT: v_mul_i32_i24_sdwa v9, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX90A-NEXT: v_mul_i32_i24_sdwa v4, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX90A-NEXT: v_mul_i32_i24_sdwa v6, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX90A-NEXT: v_mul_i32_i24_sdwa v10, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX90A-NEXT: v_mul_i32_i24_sdwa v5, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX90A-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX90A-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX90A-NEXT: v_add_u32_e32 v4, v4, v5
-; GFX90A-NEXT: v_add_u32_e32 v2, v9, v10
-; GFX90A-NEXT: v_add_u32_e32 v3, v8, v6
-; GFX90A-NEXT: v_add3_u32 v0, v0, v1, v4
-; GFX90A-NEXT: v_add3_u32 v0, v3, v2, v0
+; GFX90A-NEXT: v_dot4c_i32_i8_e32 v0, v4, v6
+; GFX90A-NEXT: v_dot4c_i32_i8_e32 v0, v5, v7
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX1030-LABEL: dot8_sext_i8:
; GFX1030: ; %bb.0: ; %entry
; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1030-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX1030-NEXT: global_load_dwordx2 v[2:3], v[2:3], off
-; GFX1030-NEXT: s_waitcnt vmcnt(1)
-; GFX1030-NEXT: v_lshrrev_b16 v4, 8, v0
-; GFX1030-NEXT: v_lshrrev_b16 v5, 8, v1
+; GFX1030-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX1030-NEXT: global_load_dwordx2 v[1:2], v[2:3], off
+; GFX1030-NEXT: v_mov_b32_e32 v0, 0
; GFX1030-NEXT: s_waitcnt vmcnt(0)
-; GFX1030-NEXT: v_lshrrev_b16 v6, 8, v2
-; GFX1030-NEXT: v_lshrrev_b16 v7, 8, v3
-; GFX1030-NEXT: v_mul_i32_i24_sdwa v8, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX1030-NEXT: v_mul_i32_i24_sdwa v9, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX1030-NEXT: v_mul_i32_i24_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX1030-NEXT: v_mul_i32_i24_sdwa v2, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX1030-NEXT: v_mul_i32_i24_sdwa v10, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX1030-NEXT: v_mul_i32_i24_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:BYTE_2
-; GFX1030-NEXT: v_mul_i32_i24_sdwa v3, sext(v4), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX1030-NEXT: v_mul_i32_i24_sdwa v4, sext(v5), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX1030-NEXT: v_add_nc_u32_e32 v0, v0, v2
-; GFX1030-NEXT: v_add_nc_u32_e32 v2, v8, v10
-; GFX1030-NEXT: v_add_nc_u32_e32 v1, v9, v1
-; GFX1030-NEXT: v_add3_u32 v0, v3, v4, v0
-; GFX1030-NEXT: v_add3_u32 v0, v2, v1, v0
+; GFX1030-NEXT: v_dot4c_i32_i8 v0, v4, v1
+; GFX1030-NEXT: v_dot4c_i32_i8 v0, v5, v2
; GFX1030-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: dot8_sext_i8:
@@ -554,39 +528,11 @@ define i32 @dot8_sext_i8(ptr addrspace(1) %a, ptr addrspace(1) %b) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[4:5], v[0:1], off
; GFX1250-NEXT: global_load_b64 v[6:7], v[2:3], off
-; GFX1250-NEXT: s_wait_loadcnt 0x1
-; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_lshrrev_b16 v0.l, 8, v4.l
-; GFX1250-NEXT: v_lshrrev_b16 v1.l, 8, v5.l
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_lshrrev_b16 v8.l, 8, v6.l
-; GFX1250-NEXT: v_lshrrev_b16 v9.l, 8, v7.l
-; GFX1250-NEXT: v_dual_ashrrev_i32 v12, 24, v5 :: v_dual_ashrrev_i32 v14, 24, v7
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_ashrrev_i32 v2, 24, v4 :: v_dual_ashrrev_i32 v10, 24, v6
-; GFX1250-NEXT: v_bfe_i32 v13, v5, 16, 8
-; GFX1250-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX1250-NEXT: v_bfe_i32 v15, v7, 16, 8
-; GFX1250-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX1250-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX1250-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX1250-NEXT: v_bfe_i32 v9, v9, 0, 8
-; GFX1250-NEXT: v_mul_i32_i24_e32 v12, v12, v14
-; GFX1250-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX1250-NEXT: v_bfe_i32 v4, v4, 16, 8
-; GFX1250-NEXT: v_bfe_i32 v11, v6, 0, 8
-; GFX1250-NEXT: v_bfe_i32 v6, v6, 16, 8
-; GFX1250-NEXT: v_mul_i32_i24_e32 v5, v5, v7
-; GFX1250-NEXT: v_mul_i32_i24_e32 v7, v13, v15
-; GFX1250-NEXT: v_mul_i32_i24_e32 v0, v0, v8
-; GFX1250-NEXT: v_mul_i32_i24_e32 v1, v1, v9
-; GFX1250-NEXT: v_mad_i32_i24 v2, v2, v10, v12
-; GFX1250-NEXT: v_mad_i32_i24 v3, v3, v11, v5
-; GFX1250-NEXT: v_mad_i32_i24 v4, v4, v6, v7
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add3_u32 v0, v0, v1, v2
-; GFX1250-NEXT: v_add3_u32 v0, v3, v4, v0
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_dot4_i32_iu8 v0, v4, v6, 0 neg_lo:[1,1,0]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_dot4_i32_iu8 v0, v5, v7, v0 neg_lo:[1,1,0]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
entry:
%pa0 = getelementptr inbounds i8, ptr addrspace(1) %a, i32 0
@@ -660,35 +606,22 @@ define i32 @sum4_zext_i8(ptr addrspace(1) %a) {
; GFX90A: ; %bb.0: ; %entry
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v0, v[0:1], off
-; GFX90A-NEXT: s_movk_i32 s4, 0xff
-; GFX90A-NEXT: s_mov_b32 s5, 0x5040100
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v1, 24, v0
-; GFX90A-NEXT: v_lshrrev_b16_e32 v2, 8, v0
-; GFX90A-NEXT: v_and_b32_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX90A-NEXT: v_perm_b32 v0, v2, v0, s5
-; GFX90A-NEXT: v_perm_b32 v1, v1, v3, s5
-; GFX90A-NEXT: v_pk_add_u16 v0, v0, v1
-; GFX90A-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: v_add_u32_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_1
+; GFX90A-NEXT: v_bfe_u32 v0, v0, 16, 8
+; GFX90A-NEXT: v_add3_u32 v0, v2, v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX1030-LABEL: sum4_zext_i8:
; GFX1030: ; %bb.0: ; %entry
; GFX1030-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX1030-NEXT: global_load_dword v0, v[0:1], off
-; GFX1030-NEXT: v_mov_b32_e32 v1, 0xff
; GFX1030-NEXT: s_waitcnt vmcnt(0)
-; GFX1030-NEXT: v_lshrrev_b32_e32 v2, 24, v0
-; GFX1030-NEXT: v_lshrrev_b16 v3, 8, v0
-; GFX1030-NEXT: v_and_b32_e32 v4, 0xff, v0
-; GFX103...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/228904
More information about the llvm-branch-commits
mailing list