[llvm] [AMDGPU]Select build_vector to v_pk_mov (PR #221673)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 7 01:06:20 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-amdgpu
Author: Shoreshen
<details>
<summary>Changes</summary>
This add selection from `build_vector` to `v_pk_mov` copy from `AMDGPUDAGToDAGISel::SelectVectorShuffle`.
For test case as follow:
```llvm
define void @<!-- -->v_shuffle_v2p3_v3p3__1_5(ptr addrspace(1) inreg %ptr) {
%vec0 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
%vec1 = call <3 x ptr addrspace(3)> asm "; def $0", "=v"()
%shuf = shufflevector <3 x ptr addrspace(3)> %vec0, <3 x ptr addrspace(3)> %vec1, <2 x i32> <i32 1, i32 5>
store <2 x ptr addrspace(3)> %shuf, ptr addrspace(1) %ptr, align 8
ret void
}
```
The pre-selection DAG would be like:
```mlir
Optimized legalized selection DAG: %bb.0 'v_shuffle_v2p3_v3p3__1_5:'
SelectionDAG has 24 nodes:
...
t12: v3i32,ch,glue = CopyFromReg # D:1 t11, Register:v3i32 %10, t11:1
t17: i32 = extract_vector_elt # D:1 t12, Constant:i32<1>
t15: v3i32,ch,glue = CopyFromReg # D:1 t14, Register:v3i32 %11, t14:1
t19: i32 = extract_vector_elt # D:1 t15, Constant:i32<2>
t20: v2i32 = BUILD_VECTOR # D:1 t17, t19
...
```
This pattern can be selected into `v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]`. And the final asm is:
```asm
s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
;;#ASMSTART
; def v[2:4]
;;#ASMEND
v_mov_b32_e32 v5, 0
;;#ASMSTART
; def v[0:2]
;;#ASMEND
s_nop 0
v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
global_store_dwordx2 v5, v[0:1], s[0:1]
s_waitcnt vmcnt(0)
s_setpc_b64 s[30:31]
```
note that compiler folded `%vec0`, `%vec1` that overlap on `v2` so that `v_pk_mov`'s source are only `v[2:3]`.
## Problem
This pattern only select `build_vector(extract_vector_elt(V1, 2 * i), extract_vector_elt(W1, 2 * j + 1))` like patterns.
The reason is probably because other patterns can be eliminated or partly eliminated by register coalescing.
Assume we have `build_vector(extract_vector_elt(V1, 2 * i + 1), extract_vector_elt(W1, 2 * j + 1))`
Instead of creating
```mlir
%vgpr1 = extract_vector_elt(V1, 2 * i + 1)
%vgpr2 = extract_vector_elt(W1, 2 * i + 1)
v[0:1] = V_PK_MOV_B32
REG_SEQUENCE(undef, %vgpr1),
REG_SEQUENCE(undef, %vgpr2),
op_sel[1,0]
```
we probably should create:
```mlir
%vgpr1 = extract_vector_elt(V1, 2 * i + 1)
%vgpr2 = extract_vector_elt(W1, 2 * i + 1)
v0 = V_MOV_B32 %vgpr1
v1 = V_MOV_B32 %vgpr2
```
Then coalesce may coalesce `v1` with the result of `extract_vector_elt(W1, 2 * j + 1)`, which will make the compiler allocate `v1` to `%vgpr2` and hence only `v0 = V_MOV_B32 extract_vector_elt(V1, 2 * i + 1)` will be left.
However, this is not garenteened to happen. If `extract_vector_elt(W1, 2 * j + 1)` is still alive after the use of `V_PK_MOV_B32`, then `v[0:1]` and `extract_vector_elt(W1, 2 * j + 1)` will both be alive, thus cannot coalesce. And one easy example will be:
```llvm
define void @<!-- -->no_coalesce(ptr addrspace(1) %out) {
%a = call <2 x i32> asm "; def $0", "=v"()
%b = call <2 x i32> asm "; def $0", "=v"()
%r = shufflevector <2 x i32> %a, <2 x i32> %b,
<2 x i32> <i32 0, i32 2>
store <2 x i32> %r, ptr addrspace(1) %out
call void asm sideeffect "; use $0", "v"(<2 x i32> %a)
call void asm sideeffect "; use $0", "v"(<2 x i32> %b)
ret void
}
```
The generated result will be:
```s
s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
;;#ASMSTART
; def v[2:3]
;;#ASMEND
;;#ASMSTART
; def v[4:5]
;;#ASMEND
s_nop 0
v_mov_b32_e32 v6, v2
v_mov_b32_e32 v7, v4
global_store_dwordx2 v[0:1], v[6:7], off
;;#ASMSTART
; use v[2:3]
;;#ASMEND
;;#ASMSTART
; use v[4:5]
;;#ASMEND
s_waitcnt vmcnt(0)
s_setpc_b64 s[30:31]
```
It can be seen that:
```
v_mov_b32_e32 v6, v2
v_mov_b32_e32 v7, v4
```
Can be replaced by `v_pk_mov_b32 v[6:7], v[2:3], v[4:5] op_sel:[0,0]`
So instead of selecting `v_pk_mov`, it is probably better to insert a pass to generate `V_PK_MOV_B32` from `V_MOV_B32` after register coalescing pass.
---
Patch is 237.08 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/221673.diff
18 Files Affected:
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp (+116)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h (+3)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v3i32.ll (+23-27)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll (+12-20)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v8i32.ll (+60-108)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v3p3.ll (+23-27)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v4p3.ll (+12-20)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v2p3.v8p3.ll (+60-108)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v3f32.v2f32.ll (+52-71)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v3f32.v4f32.ll (+16-28)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v3i32.v2i32.ll (+52-71)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v3i32.v3i32.ll (+30-40)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v3i32.v4i32.ll (+175-261)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v3p3.v2p3.ll (+52-71)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v3p3.v3p3.ll (+30-40)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v3p3.v4p3.ll (+175-261)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4i32.v2i32.ll (+6-9)
- (modified) llvm/test/CodeGen/AMDGPU/shufflevector.v4p3.v2p3.ll (+6-9)
``````````diff
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 0d212f9dd39b2..e43207f5bc8b9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -502,6 +502,79 @@ SDNode *AMDGPUDAGToDAGISel::packConstantV2I16(const SDNode *N,
return nullptr;
}
+/// Match \p Op as a 32-bit element that V_PK_MOV_B32 can read as one half of a
+/// 64-bit operand. \p HighHalf is set to the half of the aligned pair that the
+/// element occupies in the register it comes from, which is the half op_sel has
+/// to select for the REG_SEQUENCE built around it to fold away again.
+static bool matchPkMovHalf(SDValue Op, bool &HighHalf) {
+ if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+ !Op.getValueType().bitsEq(MVT::i32))
+ return false;
+
+ auto *IdxC = dyn_cast<ConstantSDNode>(Op.getOperand(1));
+ if (!IdxC)
+ return false;
+
+ SDValue Src = Op.getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ if (!SrcVT.isVector() || !SrcVT.getVectorElementType().bitsEq(MVT::i32) ||
+ !Src->isDivergent())
+ return false;
+
+ HighHalf = IdxC->getZExtValue() & 1;
+ return true;
+}
+
+SDValue AMDGPUDAGToDAGISel::buildPkMovOperand(SDValue Elt, bool HighHalf,
+ const SDLoc &DL) {
+ // Build the pair rather than extracting it out of whatever register the
+ // element lives in. The containing aligned pair need not exist there at all,
+ // as it does not for the last element of a 3-element vector. When it does
+ // exist this REG_SEQUENCE coalesces back into a plain subregister reference.
+ const SIRegisterInfo *TRI = Subtarget->getRegisterInfo();
+ SDValue Undef = SDValue(
+ CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::i32), 0);
+ const SDValue Ops[] = {
+ CurDAG->getTargetConstant(
+ TRI->getDefaultVectorSuperClassForBitWidth(64)->getID(), DL,
+ MVT::i32),
+ HighHalf ? Undef : Elt,
+ CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
+ HighHalf ? Elt : Undef,
+ CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32)};
+ return SDValue(
+ CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::v2i32, Ops),
+ 0);
+}
+
+bool AMDGPUDAGToDAGISel::matchPkMovPair(SDValue Lo, SDValue Hi, const SDLoc &DL,
+ SmallVectorImpl<SDValue> &Ops) {
+ bool LoHigh, HiHigh;
+ if (!matchPkMovHalf(Lo, LoHigh) || !matchPkMovHalf(Hi, HiHigh))
+ return false;
+
+ // The low result lane always comes from src0 and the high one from src1, so a
+ // packed move only saves anything when both elements sit in the wrong half.
+ // Otherwise a plain subregister copy is at least as cheap.
+ if (!LoHigh || HiHigh)
+ return false;
+
+ SDValue ZeroMods = CurDAG->getTargetConstant(0, DL, MVT::i32);
+ // op_sel picks the high half of src0. op_sel_hi is set on both sources purely
+ // to keep it from being printed.
+ Ops.append({CurDAG->getTargetConstant(
+ SISrcMods::OP_SEL_0 | SISrcMods::OP_SEL_1, DL, MVT::i32),
+ buildPkMovOperand(Lo, LoHigh, DL),
+ CurDAG->getTargetConstant(SISrcMods::OP_SEL_1, DL, MVT::i32),
+ buildPkMovOperand(Hi, HiHigh, DL),
+ ZeroMods, // clamp
+ ZeroMods, // op_sel
+ ZeroMods, // op_sel_hi
+ ZeroMods, // neg_lo
+ ZeroMods}); // neg_hi
+ return true;
+}
+
void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
EVT VT = N->getValueType(0);
unsigned NumVectorElts = VT.getVectorNumElements();
@@ -544,6 +617,49 @@ void AMDGPUDAGToDAGISel::SelectBuildVector(SDNode *N, unsigned RegClassID) {
}
}
+ // Two adjacent result lanes whose elements both have to cross halves can be
+ // produced by a single packed move rather than two copies. The lanes have to
+ // be an aligned pair, since that is the REG_SEQUENCE slot the move fills, and
+ // they have to be the only defined elements: packing every pair of a fully
+ // defined vector can drag values out of AGPRs, where the per-lane copies this
+ // otherwise expands to are cheaper.
+ if (IsGCN && Subtarget->hasPkMovB32() && N->isDivergent() &&
+ EltVT.getSizeInBits() == 32 && N->getNumOperands() == NumVectorElts) {
+ SmallVector<SmallVector<SDValue, 9>, 2> PairOps(NumVectorElts / 2);
+ bool AnyPacked = false;
+ if (NumVectorElts <= 4) {
+ for (unsigned I = 0, E = NumVectorElts / 2; I != E; ++I)
+ AnyPacked |= matchPkMovPair(N->getOperand(2 * I),
+ N->getOperand(2 * I + 1), DL, PairOps[I]);
+ }
+
+ if (AnyPacked) {
+ // A single pair covering the whole result needs no REG_SEQUENCE.
+ if (NumVectorElts == 2) {
+ CurDAG->SelectNodeTo(N, AMDGPU::V_PK_MOV_B32, N->getVTList(),
+ PairOps[0]);
+ return;
+ }
+
+ SmallVector<SDValue, 32 * 2 + 1> Args = {RegClass};
+ for (unsigned I = 0; I != NumVectorElts;) {
+ bool UsePacked =
+ I % 2 == 0 && I / 2 < PairOps.size() && !PairOps[I / 2].empty();
+ unsigned NumRegs = UsePacked ? 2 : 1;
+ Args.push_back(UsePacked ? SDValue(CurDAG->getMachineNode(
+ AMDGPU::V_PK_MOV_B32, DL,
+ MVT::v2i32, PairOps[I / 2]),
+ 0)
+ : N->getOperand(I));
+ Args.push_back(CurDAG->getTargetConstant(
+ SIRegisterInfo::getSubRegFromChannel(I, NumRegs), DL, MVT::i32));
+ I += NumRegs;
+ }
+ CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, N->getVTList(), Args);
+ return;
+ }
+ }
+
assert(NumVectorElts <= 32 && "Vectors with more than 32 elements not "
"supported yet");
// 32 = Max Num Vector Elements
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
index 2bcfe530b3947..866bf08598955 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
@@ -73,6 +73,9 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
protected:
void SelectBuildVector(SDNode *N, unsigned RegClassID);
+ SDValue buildPkMovOperand(SDValue Elt, bool HighHalf, const SDLoc &DL);
+ bool matchPkMovPair(SDValue Lo, SDValue Hi, const SDLoc &DL,
+ SmallVectorImpl<SDValue> &Ops);
void SelectVectorShuffle(SDNode *N);
bool isSDWAOperand(const SDNode *N) const;
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v3i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v3i32.ll
index 23f6e612ef2be..cd4e6e64ee47c 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v3i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v3i32.ll
@@ -1127,12 +1127,11 @@ define void @v_shuffle_v2i32_v3i32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v2i32_v3i32__1_2:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v3, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1140,12 +1139,12 @@ define void @v_shuffle_v2i32_v3i32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-LABEL: v_shuffle_v2i32_v3i32__1_2:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v3, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v3, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1258,15 +1257,14 @@ define void @v_shuffle_v2i32_v3i32__4_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v2i32_v3i32__4_2:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[4:6]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v0, v5
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v3, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1274,15 +1272,15 @@ define void @v_shuffle_v2i32_v3i32__4_2(ptr addrspace(1) inreg %ptr) {
; GFX942-LABEL: v_shuffle_v2i32_v3i32__4_2:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v3, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v3, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[4:6]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
-; GFX942-NEXT: v_mov_b32_e32 v0, v5
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1874,14 +1872,13 @@ define void @v_shuffle_v2i32_v3i32__1_5(ptr addrspace(1) inreg %ptr) {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[0:2]
+; GFX90A-NEXT: ; def v[2:4]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v[2:4]
+; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v4
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v5, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1890,14 +1887,14 @@ define void @v_shuffle_v2i32_v3i32__1_5(ptr addrspace(1) inreg %ptr) {
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[0:2]
+; GFX942-NEXT: ; def v[2:4]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v5, 0
; GFX942-NEXT: ;;#ASMSTART
-; GFX942-NEXT: ; def v[2:4]
+; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v5, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -2019,12 +2016,11 @@ define void @v_shuffle_v2i32_v3i32__4_5(ptr addrspace(1) inreg %ptr) {
; GFX90A-LABEL: v_shuffle_v2i32_v3i32__4_5:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v3, 0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v[0:2]
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v3, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -2032,12 +2028,12 @@ define void @v_shuffle_v2i32_v3i32__4_5(ptr addrspace(1) inreg %ptr) {
; GFX942-LABEL: v_shuffle_v2i32_v3i32__4_5:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v3, 0
; GFX942-NEXT: ;;#ASMSTART
; GFX942-NEXT: ; def v[0:2]
; GFX942-NEXT: ;;#ASMEND
-; GFX942-NEXT: v_mov_b32_e32 v3, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
index 23132713a076e..47cc3b77d6035 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v4i32.ll
@@ -538,9 +538,8 @@ define void @v_shuffle_v2i32_v4i32__7_4(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -551,9 +550,8 @@ define void @v_shuffle_v2i32_v4i32__7_4(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -879,9 +877,8 @@ define void @v_shuffle_v2i32_v4i32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v4, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -892,9 +889,8 @@ define void @v_shuffle_v2i32_v4i32__3_0(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v4, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <4 x i32> asm "; def $0", "=v"()
@@ -1501,8 +1497,7 @@ define void @v_shuffle_v2i32_v4i32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1514,8 +1509,7 @@ define void @v_shuffle_v2i32_v4i32__1_2(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -3067,8 +3061,7 @@ define void @v_shuffle_v2i32_v4i32__5_6(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:3]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v4, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v4, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -3080,8 +3073,7 @@ define void @v_shuffle_v2i32_v4i32__5_6(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:3]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v4, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[0:1], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v8i32.ll b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v8i32.ll
index ee8f880b6ab50..17577fcf8549c 100644
--- a/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v8i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/shufflevector.v2i32.v8i32.ll
@@ -1070,9 +1070,8 @@ define void @v_shuffle_v2i32_v8i32__15_8(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v2, v7
-; GFX90A-NEXT: v_mov_b32_e32 v3, v0
-; GFX90A-NEXT: global_store_dwordx2 v8, v[2:3], s[16:17]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -1083,9 +1082,8 @@ define void @v_shuffle_v2i32_v8i32__15_8(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, v7
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: global_store_dwordx2 v8, v[2:3], s[0:1]
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[0:1] op_sel:[1,0]
+; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
%vec0 = call <8 x i32> asm "; def $0", "=v"()
@@ -1158,8 +1156,7 @@ define void @v_shuffle_v2i32_v8i32__15_10(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v7
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1171,8 +1168,7 @@ define void @v_shuffle_v2i32_v8i32__15_10(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v7
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[2:3] op_sel:[1,0]
; GFX942-NEXT: global_store_dwordx2 v8, v[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
@@ -1246,8 +1242,7 @@ define void @v_shuffle_v2i32_v8i32__15_12(ptr addrspace(1) inreg %ptr) {
; GFX90A-NEXT: ; def v[0:7]
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_mov_b32_e32 v8, 0
-; GFX90A-NEXT: v_mov_b32_e32 v0, v7
-; GFX90A-NEXT: v_mov_b32_e32 v1, v4
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[4:5] op_sel:[1,0]
; GFX90A-NEXT: global_store_dwordx2 v8, v[0:1], s[16:17]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
@@ -1259,8 +1254,7 @@ define void @v_shuffle_v2i32_v8i32__15_12(ptr addrspace(1) inreg %ptr) {
; GFX942-NEXT: ; def v[0:7]
; GFX942-NEXT: ;;#ASMEND
; GFX942-NEXT: v_mov_b32_e32 v8, 0
-; GFX942-NEXT: v_mov_b32_e32 v0, v7
-; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX94...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/221673
More information about the llvm-commits
mailing list