[llvm] [AMDGPU] Re-fold masked hi16 pack into v_or_b32_sdwa (PR #210735)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 20 08:11:16 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: michaelselehov
<details>
<summary>Changes</summary>
Follow-up to #<!-- -->206058, which selects the (hi << 16) | (z & 0xffff) high-half pack to a fused v_lshl_or_b32. When z's high bits are not known zero this needs an explicit v_and_b32 0xffff, and since v_lshl_or_b32 has no SDWA form the mask can no longer fold into a single v_or_b32_sdwa (src1_sel:WORD_0) as it did before -- a small but real perf hole (~5% on a VALU-bound char4 kernel on gfx942).
Add a SIPeepholeSDWA rewrite of the masked pack back into v_lshlrev_b32 + v_or_b32_sdwa, erasing the dead v_and. It fires only on the masked form, so the mask-free #<!-- -->206058 win is preserved, and it never increases the instruction count.
Assisted-by: Claude Opus
---
Patch is 1.60 MiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/210735.diff
70 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp (+109)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll (+93-99)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll (+17-21)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll (+4-4)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll (+106-796)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll (+52-386)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.getresinfo.a16.ll (+16-144)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll (+57-37)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll (+63-43)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.cd.g16.ll (+29-32)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll (+52-52)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll (+41-47)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/select-to-fmin-fmax.ll (+24-24)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll (+41-47)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll (+41-47)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll (+41-47)
- (modified) llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll (+4-4)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll (+2686-2748)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll (+186-191)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll (+469-495)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll (+146-164)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll (+14-18)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll (+7-11)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll (+942-1007)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.576bit.ll (+216-216)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.640bit.ll (+240-240)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll (+74-79)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.704bit.ll (+264-264)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.768bit.ll (+288-288)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.832bit.ll (+312-312)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.896bit.ll (+336-336)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.960bit.ll (+360-360)
- (modified) llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll (+78-81)
- (modified) llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll (+74-86)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll (+202-202)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll (+76-76)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll (+76-76)
- (modified) llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll (+4-6)
- (modified) llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll (+12-12)
- (modified) llvm/test/CodeGen/AMDGPU/constant-address-space-32bit.ll (+3-4)
- (modified) llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll (+15-19)
- (modified) llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll (+13-13)
- (modified) llvm/test/CodeGen/AMDGPU/fmaximum.ll (+20-25)
- (modified) llvm/test/CodeGen/AMDGPU/fminimum.ll (+20-25)
- (modified) llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll (+20-30)
- (modified) llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll (+7-10)
- (modified) llvm/test/CodeGen/AMDGPU/fneg.bf16.ll (+18-19)
- (modified) llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll (+24-31)
- (modified) llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll (+27-35)
- (modified) llvm/test/CodeGen/AMDGPU/freeze.ll (+10-10)
- (modified) llvm/test/CodeGen/AMDGPU/frem.ll (+14-18)
- (modified) llvm/test/CodeGen/AMDGPU/idot4u.ll (+16-17)
- (modified) llvm/test/CodeGen/AMDGPU/idot8s.ll (+127-133)
- (modified) llvm/test/CodeGen/AMDGPU/idot8u.ll (+99-103)
- (modified) llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll (+4-4)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.a16.dim.ll (+153-160)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.noret.ll (+3-4)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.fptrunc.round.ll (+18-18)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll (+5-7)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll (+5-7)
- (modified) llvm/test/CodeGen/AMDGPU/llvm.modf.ll (+4-6)
- (modified) llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll (+8-8)
- (modified) llvm/test/CodeGen/AMDGPU/packed-op-sel.ll (+5-7)
- (modified) llvm/test/CodeGen/AMDGPU/permute_i8.ll (+267-294)
- (modified) llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll (+4-5)
- (added) llvm/test/CodeGen/AMDGPU/sdwa-peephole-lshl-or-hi16pack.mir (+156)
- (modified) llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll (+36-43)
- (modified) llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll (+6-6)
- (modified) llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll (+7-7)
- (modified) llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll (+11-11)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index 8c45017cd4c43..9262b004aefb6 100644
--- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -66,6 +66,7 @@ class SIPeepholeSDWA {
MachineInstr *createSDWAVersion(MachineInstr &MI);
bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
void legalizeScalarOperands(MachineInstr &MI, const GCNSubtarget &ST) const;
+ bool splitLshlOrForSDWA(MachineBasicBlock &MBB);
public:
bool run(MachineFunction &MF);
@@ -1359,6 +1360,108 @@ void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
}
}
+// Rewrite the high-half packing idiom
+// %m = V_AND_B32 0xffff, %z ; (single use)
+// %d = V_LSHL_OR_B32 %hi, 16, %m ; (%hi << 16) | (%z & 0xffff)
+// into
+// %s = V_LSHLREV_B32 16, %hi
+// %d = V_OR_B32_sdwa %s, %z src1_sel:WORD_0
+// and erase the now-dead V_AND. The fused V_LSHL_OR_B32 has no SDWA form
+// (getSDWAOp == -1), so once ISel emits it the regular peephole can no longer
+// fold the 0xffff mask into V_OR_B32_sdwa; this restores that fold.
+//
+// The rewrite is done atomically (the sdwa OR is built directly and the V_AND
+// erased) rather than by splitting and hoping the generic matcher re-folds:
+// if the V_AND were left in place a competing fold (e.g. folding a source
+// V_LSHRREV into the V_AND) could win and leave the extra V_LSHLREV behind,
+// increasing the instruction count. Built this way the count is never worse
+// (2 ops -> 2 ops in isolation; the following MachineCSE shares the %hi<<16
+// shift when it already exists, and the generic matcher folds any source
+// select of %z into the new sdwa OR, giving the pre-#206058 codegen).
+bool SIPeepholeSDWA::splitLshlOrForSDWA(MachineBasicBlock &MBB) {
+ struct Candidate {
+ MachineInstr *LshlOr;
+ MachineInstr *AndMI;
+ MachineOperand *Hi; // V_LSHL_OR_B32 src0 (high half)
+ MachineOperand *ValSrc; // the non-0xffff operand of the V_AND
+ };
+ SmallVector<Candidate, 4> Candidates;
+
+ for (MachineInstr &MI : MBB) {
+ if (MI.getOpcode() != AMDGPU::V_LSHL_OR_B32_e64)
+ continue;
+
+ // Shift amount must be 16 (packing into the high 16 bits).
+ MachineOperand *Shift = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
+ std::optional<int64_t> ShiftImm = foldToImm(*Shift);
+ if (!ShiftImm || *ShiftImm != 16)
+ continue;
+
+ // vdst is copied into the new V_OR_B32_sdwa; it must be a virtual VGPR
+ // (guaranteed pre-RA, checked for consistency with the SDWA matchers).
+ MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
+ if (!Dst->isReg() || Dst->getReg().isPhysical())
+ continue;
+
+ MachineOperand *Hi = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+ MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
+ if (!Hi->isReg() || Hi->getReg().isPhysical() || !Src2->isReg() ||
+ Src2->getReg().isPhysical())
+ continue;
+
+ // Src2 must be (V_AND_B32 0xffff, %z), single-use, so it can be replaced
+ // by reading %z's WORD_0 and the V_AND removed.
+ if (!MRI->hasOneNonDBGUse(Src2->getReg()))
+ continue;
+ MachineInstr *AndMI = MRI->getVRegDef(Src2->getReg());
+ if (!AndMI || (AndMI->getOpcode() != AMDGPU::V_AND_B32_e32 &&
+ AndMI->getOpcode() != AMDGPU::V_AND_B32_e64))
+ continue;
+ MachineOperand *A0 = TII->getNamedOperand(*AndMI, AMDGPU::OpName::src0);
+ MachineOperand *A1 = TII->getNamedOperand(*AndMI, AMDGPU::OpName::src1);
+ std::optional<int64_t> M0 = foldToImm(*A0);
+ std::optional<int64_t> M1 = foldToImm(*A1);
+ MachineOperand *ValSrc =
+ (M0 && *M0 == 0xffff) ? A1 : ((M1 && *M1 == 0xffff) ? A0 : nullptr);
+ if (!ValSrc || !ValSrc->isReg() || ValSrc->getReg().isPhysical() ||
+ !TRI->isVGPR(*MRI, ValSrc->getReg()))
+ continue;
+
+ Candidates.push_back({&MI, AndMI, Hi, ValSrc});
+ }
+
+ for (const Candidate &C : Candidates) {
+ MachineOperand *Dst = TII->getNamedOperand(*C.LshlOr, AMDGPU::OpName::vdst);
+
+ Register ShiftReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+ BuildMI(*C.LshlOr->getParent(), *C.LshlOr, C.LshlOr->getDebugLoc(),
+ TII->get(AMDGPU::V_LSHLREV_B32_e64), ShiftReg)
+ .addImm(16)
+ .add(*C.Hi);
+
+ // V_OR_B32_sdwa %d, 0, %s, 0, %z, clamp, dst_sel:DWORD, UNUSED_PAD,
+ // src0_sel:DWORD, src1_sel:WORD_0
+ BuildMI(*C.LshlOr->getParent(), *C.LshlOr, C.LshlOr->getDebugLoc(),
+ TII->get(AMDGPU::V_OR_B32_sdwa))
+ .add(*Dst)
+ .addImm(0)
+ .addReg(ShiftReg)
+ .addImm(0)
+ .add(*C.ValSrc)
+ .addImm(0)
+ .addImm(DWORD)
+ .addImm(UNUSED_PAD)
+ .addImm(DWORD)
+ .addImm(WORD_0);
+
+ MRI->clearKillFlags(C.ValSrc->getReg());
+ C.LshlOr->eraseFromParent();
+ C.AndMI->eraseFromParent();
+ }
+
+ return !Candidates.empty();
+}
+
bool SIPeepholeSDWALegacy::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
@@ -1381,6 +1484,12 @@ bool SIPeepholeSDWA::run(MachineFunction &MF) {
for (MachineBasicBlock &MBB : MF) {
bool Changed = false;
do {
+ // Split fused V_LSHL_OR_B32 high-half packs whose low operand is a
+ // maskable (V_AND 0xffff) value, so the mask can be folded into a
+ // single V_OR_B32_sdwa below.
+ if (splitLshlOrForSDWA(MBB))
+ Ret = true;
+
// Preprocess the ADD/SUB pairs so they could be SDWA'ed.
// Look for a possible ADD or SUB that resulted from a previously lowered
// V_{ADD|SUB}_U64_PSEUDO. The function pseudoOpConvertToVOP2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
index 3a8d2a90a7a84..d3eed3e7b4fd9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
@@ -43,31 +43,30 @@ define void @add_v3i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-LABEL: add_v3i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v6, v[0:1], off
+; GFX9-NEXT: global_load_ushort v6, v[0:1], off offset:2
; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:4
-; GFX9-NEXT: global_load_ushort v8, v[2:3], off
+; GFX9-NEXT: global_load_ushort v8, v[2:3], off offset:2
; GFX9-NEXT: global_load_ushort v9, v[2:3], off offset:4
-; GFX9-NEXT: global_load_ushort v10, v[0:1], off offset:2
-; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:2
+; GFX9-NEXT: global_load_ushort v10, v[0:1], off
+; GFX9-NEXT: global_load_ushort v11, v[2:3], off
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; GFX9-NEXT: v_or_b32_sdwa v2, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v0, v10, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v2, v11, 16, v2
-; GFX9-NEXT: v_lshl_or_b32 v1, s4, 16, v1
-; GFX9-NEXT: v_lshl_or_b32 v3, s4, 16, v3
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_add_u16 v1, v1, v3
-; GFX9-NEXT: global_store_short v[4:5], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:2
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:4
+; GFX9-NEXT: v_pk_add_u16 v0, v2, v0
+; GFX9-NEXT: global_store_short v[4:5], v1, off
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:2
+; GFX9-NEXT: global_store_short v[4:5], v0, off offset:4
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <3 x i16>, ptr addrspace(1) %ptra, align 4
@@ -213,46 +212,45 @@ define void @add_v5i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-LABEL: add_v5i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v6, v[0:1], off
-; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:4
+; GFX9-NEXT: global_load_ushort v6, v[0:1], off offset:2
+; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:6
; GFX9-NEXT: global_load_ushort v8, v[0:1], off offset:8
-; GFX9-NEXT: global_load_ushort v9, v[2:3], off
-; GFX9-NEXT: global_load_ushort v10, v[2:3], off offset:4
+; GFX9-NEXT: global_load_ushort v9, v[2:3], off offset:2
+; GFX9-NEXT: global_load_ushort v10, v[2:3], off offset:6
; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:8
-; GFX9-NEXT: global_load_ushort v12, v[0:1], off offset:2
-; GFX9-NEXT: global_load_ushort v13, v[0:1], off offset:6
-; GFX9-NEXT: global_load_ushort v14, v[2:3], off offset:2
-; GFX9-NEXT: global_load_ushort v15, v[2:3], off offset:6
+; GFX9-NEXT: global_load_ushort v12, v[0:1], off
+; GFX9-NEXT: global_load_ushort v13, v[0:1], off offset:4
+; GFX9-NEXT: global_load_ushort v14, v[2:3], off
+; GFX9-NEXT: global_load_ushort v15, v[2:3], off offset:4
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v7
; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX9-NEXT: v_or_b32_sdwa v3, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v6, 16, v9
; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v11
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshl_or_b32 v0, v12, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v1, v13, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v3, v14, 16, v3
-; GFX9-NEXT: v_lshl_or_b32 v2, s4, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v6, v6, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v6, v15, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v7, s4, 16, v7
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v3
+; GFX9-NEXT: v_or_b32_sdwa v7, v7, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_add_u16 v1, v1, v6
+; GFX9-NEXT: v_pk_add_u16 v0, v3, v0
; GFX9-NEXT: v_pk_add_u16 v2, v2, v7
-; GFX9-NEXT: global_store_short v[4:5], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:2
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:4
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:6
-; GFX9-NEXT: global_store_short v[4:5], v2, off offset:8
+; GFX9-NEXT: global_store_short v[4:5], v1, off
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:2
+; GFX9-NEXT: global_store_short v[4:5], v2, off offset:4
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v2, off offset:6
+; GFX9-NEXT: global_store_short v[4:5], v0, off offset:8
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <5 x i16>, ptr addrspace(1) %ptra, align 4
@@ -432,61 +430,60 @@ define void @addv_7i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-LABEL: addv_7i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v6, v[0:1], off
-; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:4
-; GFX9-NEXT: global_load_ushort v8, v[0:1], off offset:8
+; GFX9-NEXT: global_load_ushort v6, v[0:1], off offset:2
+; GFX9-NEXT: global_load_ushort v7, v[0:1], off offset:6
+; GFX9-NEXT: global_load_ushort v8, v[0:1], off offset:10
; GFX9-NEXT: global_load_ushort v9, v[0:1], off offset:12
-; GFX9-NEXT: global_load_ushort v10, v[2:3], off
-; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:4
-; GFX9-NEXT: global_load_ushort v12, v[2:3], off offset:8
+; GFX9-NEXT: global_load_ushort v10, v[2:3], off offset:2
+; GFX9-NEXT: global_load_ushort v11, v[2:3], off offset:6
+; GFX9-NEXT: global_load_ushort v12, v[2:3], off offset:10
; GFX9-NEXT: global_load_ushort v13, v[2:3], off offset:12
-; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:2
-; GFX9-NEXT: global_load_ushort v15, v[0:1], off offset:6
-; GFX9-NEXT: global_load_ushort v16, v[0:1], off offset:10
-; GFX9-NEXT: global_load_ushort v17, v[2:3], off offset:2
-; GFX9-NEXT: global_load_ushort v18, v[2:3], off offset:6
-; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:10
+; GFX9-NEXT: global_load_ushort v14, v[0:1], off
+; GFX9-NEXT: global_load_ushort v15, v[0:1], off offset:4
+; GFX9-NEXT: global_load_ushort v16, v[0:1], off offset:8
+; GFX9-NEXT: global_load_ushort v17, v[2:3], off
+; GFX9-NEXT: global_load_ushort v18, v[2:3], off offset:4
+; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:8
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(13)
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v6
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v6
; GFX9-NEXT: s_waitcnt vmcnt(12)
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v7
; GFX9-NEXT: s_waitcnt vmcnt(11)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v8
; GFX9-NEXT: s_waitcnt vmcnt(10)
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX9-NEXT: v_or_b32_sdwa v6, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(9)
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v10
+; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v10
; GFX9-NEXT: s_waitcnt vmcnt(8)
-; GFX9-NEXT: v_and_b32_e32 v7, 0xffff, v11
+; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v11
; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_and_b32_e32 v8, 0xffff, v12
+; GFX9-NEXT: v_lshlrev_b32_e32 v9, 16, v12
; GFX9-NEXT: s_waitcnt vmcnt(6)
-; GFX9-NEXT: v_and_b32_e32 v9, 0xffff, v13
+; GFX9-NEXT: v_or_b32_sdwa v0, v0, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(5)
-; GFX9-NEXT: v_lshl_or_b32 v0, v14, 16, v0
+; GFX9-NEXT: v_or_b32_sdwa v1, v1, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(4)
-; GFX9-NEXT: v_lshl_or_b32 v1, v15, 16, v1
+; GFX9-NEXT: v_or_b32_sdwa v2, v2, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(3)
-; GFX9-NEXT: v_lshl_or_b32 v2, v16, 16, v2
+; GFX9-NEXT: v_or_b32_sdwa v3, v3, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v6, v17, 16, v6
-; GFX9-NEXT: v_lshl_or_b32 v3, s4, 16, v3
+; GFX9-NEXT: v_or_b32_sdwa v7, v7, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_lshl_or_b32 v7, v18, 16, v7
+; GFX9-NEXT: v_or_b32_sdwa v8, v8, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_or_b32 v8, v19, 16, v8
-; GFX9-NEXT: v_lshl_or_b32 v9, s4, 16, v9
-; GFX9-NEXT: v_pk_add_u16 v0, v0, v6
+; GFX9-NEXT: v_or_b32_sdwa v9, v9, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: v_pk_add_u16 v1, v1, v7
+; GFX9-NEXT: v_pk_add_u16 v0, v6, v0
; GFX9-NEXT: v_pk_add_u16 v2, v2, v8
; GFX9-NEXT: v_pk_add_u16 v3, v3, v9
-; GFX9-NEXT: global_store_short v[4:5], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v0, off offset:2
-; GFX9-NEXT: global_store_short v[4:5], v1, off offset:4
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:6
-; GFX9-NEXT: global_store_short v[4:5], v2, off offset:8
-; GFX9-NEXT: global_store_short_d16_hi v[4:5], v2, off offset:10
-; GFX9-NEXT: global_store_short v[4:5], v3, off offset:12
+; GFX9-NEXT: global_store_short v[4:5], v1, off
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v1, off offset:2
+; GFX9-NEXT: global_store_short v[4:5], v2, off offset:4
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v2, off offset:6
+; GFX9-NEXT: global_store_short v[4:5], v3, off offset:8
+; GFX9-NEXT: global_store_short_d16_hi v[4:5], v3, off offset:10
+; GFX9-NEXT: global_store_short v[4:5], v0, off offset:12
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
%a = load <7 x i16>, ptr addrspace(1) %ptra, align 4
@@ -565,18 +562,17 @@ define void @add_v9i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:16
; GFX9-NEXT: global_load_ushort v15, v[2:3], off offset:16
; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(2)
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: v_or_b32_sdwa v14, v0, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-NEXT: v_or_b32_sdwa v15, v0, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_pk_add_u16 v0, v6, v10
; GFX9-NEXT: v_pk_add_u16 v1, v7, v11
; GFX9-NEXT: v_pk_add_u16 v2, v8, v12
; GFX9-NEXT: v_pk_add_u16 v3, v9, v13
-; GFX9-NEXT: v_lshl_or_b32 v6, s4, 16, v14
-; GFX9-NEXT: v_lshl_or_b32 v7, s4, 16, v15
-; GFX9-NEXT: v_pk_add_u16 v6, v6, v7
+; GFX9-NEXT: v_pk_add_u16 v6, v14, v15
; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX9-NEXT: global_store_short v[4:5], v6, off offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -740,40 +736,38 @@ define void @add_v11i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addr
; GFX9-LABEL: add_v11i16:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:16
+; GFX9-NEXT: global_load_ushort v14, v[0:1], off offset:18
; GFX9-NEXT: global_load_ushort v15, v[0:1], off offset:20
; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off
-; GFX9-NEXT: global_load_ushort v16, v[2:3], off offset:16
+; GFX9-NEXT: global_load_ushort v16, v[2:3], off offset:18
; GFX9-NEXT: global_load_ushort v17, v[2:3], off offset:20
; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off
-; GFX9-NEXT: global_load_ushort v18, v[0:1], off offset:18
-; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:18
+; GFX9-NEXT: global_load_ushort v18, v[0:1], off offset:16
+; GFX9-NEXT: global_load_ushort v19, v[2:3], off offset:16
+; GFX9-NEXT: v_lshlrev_b32_e64 v0, 16, s4
; GFX9-NEXT: s_waitcnt vmcnt(7)
-; GFX9-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NEXT: v_lshlrev_b32_e32 v14, 16, v14
...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/210735
More information about the llvm-commits
mailing list