[llvm] [X86] Fold MOVAPS+SHUFPS into PSHUFD to eliminate redundant copies when … (PR #212626)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 28 20:12:29 PDT 2026
https://github.com/AZero13 updated https://github.com/llvm/llvm-project/pull/212626
>From 9b5fddb73be1d49d06ac763dbf5afd4799c5925e Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Tue, 28 Jul 2026 16:39:58 -0400
Subject: [PATCH 1/2] Pre-commit tests (NFC)
---
.../CodeGen/X86/fixup-shufps-to-pshufd.ll | 176 ++++++++++++++++++
1 file changed, 176 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/fixup-shufps-to-pshufd.ll
diff --git a/llvm/test/CodeGen/X86/fixup-shufps-to-pshufd.ll b/llvm/test/CodeGen/X86/fixup-shufps-to-pshufd.ll
new file mode 100644
index 0000000000000..e258285700418
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fixup-shufps-to-pshufd.ll
@@ -0,0 +1,176 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE4
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -mattr=+no-bypass-delay-shuffle | FileCheck %s --check-prefixes=NO-DELAY
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver2 -mattr=-avx | FileCheck %s --check-prefixes=ZNVER2
+
+; Test that single-source v4f32 shuffles use PSHUFD instead of SHUFPS on CPUs
+; where there is no domain delay for shuffles.
+
+define <4 x float> @shuffle_v4f32_splat2(<4 x float> %a) {
+; SSE2-LABEL: shuffle_v4f32_splat2:
+; SSE2: # %bb.0:
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; SSE2-NEXT: retq
+;
+; SSE4-LABEL: shuffle_v4f32_splat2:
+; SSE4: # %bb.0:
+; SSE4-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; SSE4-NEXT: retq
+;
+; NO-DELAY-LABEL: shuffle_v4f32_splat2:
+; NO-DELAY: # %bb.0:
+; NO-DELAY-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; NO-DELAY-NEXT: retq
+;
+; ZNVER2-LABEL: shuffle_v4f32_splat2:
+; ZNVER2: # %bb.0:
+; ZNVER2-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,2,2,2]
+; ZNVER2-NEXT: retq
+ %r = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+ ret <4 x float> %r
+}
+
+define <4 x float> @shuffle_v4f32_0123(<4 x float> %a) {
+; SSE2-LABEL: shuffle_v4f32_0123:
+; SSE2: # %bb.0:
+; SSE2-NEXT: retq
+;
+; SSE4-LABEL: shuffle_v4f32_0123:
+; SSE4: # %bb.0:
+; SSE4-NEXT: retq
+;
+; NO-DELAY-LABEL: shuffle_v4f32_0123:
+; NO-DELAY: # %bb.0:
+; NO-DELAY-NEXT: retq
+;
+; ZNVER2-LABEL: shuffle_v4f32_0123:
+; ZNVER2: # %bb.0:
+; ZNVER2-NEXT: retq
+ %r = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ ret <4 x float> %r
+}
+
+define <4 x float> @shuffle_v4f32_3210(<4 x float> %a) {
+; SSE2-LABEL: shuffle_v4f32_3210:
+; SSE2: # %bb.0:
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,2,1,0]
+; SSE2-NEXT: retq
+;
+; SSE4-LABEL: shuffle_v4f32_3210:
+; SSE4: # %bb.0:
+; SSE4-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,2,1,0]
+; SSE4-NEXT: retq
+;
+; NO-DELAY-LABEL: shuffle_v4f32_3210:
+; NO-DELAY: # %bb.0:
+; NO-DELAY-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,2,1,0]
+; NO-DELAY-NEXT: retq
+;
+; ZNVER2-LABEL: shuffle_v4f32_3210:
+; ZNVER2: # %bb.0:
+; ZNVER2-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,2,1,0]
+; ZNVER2-NEXT: retq
+ %r = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+ ret <4 x float> %r
+}
+
+define <4 x float> @shuffle_v4f32_1032(<4 x float> %a) {
+; SSE2-LABEL: shuffle_v4f32_1032:
+; SSE2: # %bb.0:
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0,3,2]
+; SSE2-NEXT: retq
+;
+; SSE4-LABEL: shuffle_v4f32_1032:
+; SSE4: # %bb.0:
+; SSE4-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0,3,2]
+; SSE4-NEXT: retq
+;
+; NO-DELAY-LABEL: shuffle_v4f32_1032:
+; NO-DELAY: # %bb.0:
+; NO-DELAY-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0,3,2]
+; NO-DELAY-NEXT: retq
+;
+; ZNVER2-LABEL: shuffle_v4f32_1032:
+; ZNVER2: # %bb.0:
+; ZNVER2-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0,3,2]
+; ZNVER2-NEXT: retq
+ %r = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ ret <4 x float> %r
+}
+
+; This is a two-source shuffle — should NOT be folded to PSHUFD.
+define <4 x float> @shuffle_v4f32_two_source(<4 x float> %a, <4 x float> %b) {
+; SSE2-LABEL: shuffle_v4f32_two_source:
+; SSE2: # %bb.0:
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: retq
+;
+; SSE4-LABEL: shuffle_v4f32_two_source:
+; SSE4: # %bb.0:
+; SSE4-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE4-NEXT: retq
+;
+; NO-DELAY-LABEL: shuffle_v4f32_two_source:
+; NO-DELAY: # %bb.0:
+; NO-DELAY-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; NO-DELAY-NEXT: retq
+;
+; ZNVER2-LABEL: shuffle_v4f32_two_source:
+; ZNVER2: # %bb.0:
+; ZNVER2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; ZNVER2-NEXT: retq
+ %r = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+ ret <4 x float> %r
+}
+
+; Test where the shuffle result is used alongside the original value,
+; forcing register allocation to insert a MOVAPS copy with SHUFPS.
+; With PSHUFD, no copy is needed since PSHUFD is non-destructive.
+define void @shuffle_v4f32_with_liveout(<4 x float> %a, ptr %out1, ptr %out2) {
+; SSE2-LABEL: shuffle_v4f32_with_liveout:
+; SSE2: # %bb.0:
+; SSE2-NEXT: addps %xmm0, %xmm0
+; SSE2-NEXT: movaps %xmm0, %xmm1
+; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,2],xmm0[2,2]
+; SSE2-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: movaps %xmm1, (%rdi)
+; SSE2-NEXT: movaps %xmm0, (%rsi)
+; SSE2-NEXT: retq
+;
+; SSE4-LABEL: shuffle_v4f32_with_liveout:
+; SSE4: # %bb.0:
+; SSE4-NEXT: addps %xmm0, %xmm0
+; SSE4-NEXT: movaps %xmm0, %xmm1
+; SSE4-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,2],xmm0[2,2]
+; SSE4-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE4-NEXT: movaps %xmm1, (%rdi)
+; SSE4-NEXT: movaps %xmm0, (%rsi)
+; SSE4-NEXT: retq
+;
+; NO-DELAY-LABEL: shuffle_v4f32_with_liveout:
+; NO-DELAY: # %bb.0:
+; NO-DELAY-NEXT: addps %xmm0, %xmm0
+; NO-DELAY-NEXT: movaps %xmm0, %xmm1
+; NO-DELAY-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,2],xmm0[2,2]
+; NO-DELAY-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; NO-DELAY-NEXT: movaps %xmm1, (%rdi)
+; NO-DELAY-NEXT: movaps %xmm0, (%rsi)
+; NO-DELAY-NEXT: retq
+;
+; ZNVER2-LABEL: shuffle_v4f32_with_liveout:
+; ZNVER2: # %bb.0:
+; ZNVER2-NEXT: addps %xmm0, %xmm0
+; ZNVER2-NEXT: movaps %xmm0, %xmm1
+; ZNVER2-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,2],xmm0[2,2]
+; ZNVER2-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; ZNVER2-NEXT: movaps %xmm1, (%rdi)
+; ZNVER2-NEXT: movaps %xmm0, (%rsi)
+; ZNVER2-NEXT: retq
+ %mul = fmul <4 x float> %a, <float 2.0, float 2.0, float 2.0, float 2.0>
+ %shuf = shufflevector <4 x float> %mul, <4 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+ %add = fadd <4 x float> %shuf, <float 5.0, float 5.0, float 5.0, float 5.0>
+ store <4 x float> %add, ptr %out1
+ store <4 x float> %mul, ptr %out2
+ ret void
+}
>From c3aeccada6c1deb10c4f4097082f3650dfc54a2a Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Tue, 28 Jul 2026 17:14:57 -0400
Subject: [PATCH 2/2] [X86] Fold MOVAPS+SHUFPS into PSHUFD to eliminate
redundant copies when profitable
The Problem: When performing single-source shuffles on <4 x float> vectors using <AVX targets, LLVM relies on SHUFPS. Because SHUFPS is a 2-address destructive instruction, Instruction Selection and Register Allocation frequently have to emit a preceding 128-bit copy (MOVAPS, etc.) to preserve the original source value.
---
llvm/lib/Target/X86/X86.td | 13 +++-
llvm/lib/Target/X86/X86FixupInstTuning.cpp | 59 +++++++++++++++++++
llvm/lib/Target/X86/X86Subtarget.h | 1 +
.../CodeGen/X86/fixup-shufps-to-pshufd.ll | 10 ++--
4 files changed, 75 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index ab27fb0b6aa10..0e9da0797c332 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -688,6 +688,13 @@ def TuningNoDomainDelayShuffle : SubtargetFeature<"no-bypass-delay-shuffle",
"Has no bypass delay when using the 'wrong' shuffle type",
[], InlineIgnore>;
+// AMD processors (bdver and znver) consider any fp data movement within
+// a register (except blends) to be integer domain.
+def TuningIntDomainFloatShuffle : SubtargetFeature<"int-domain-float-shuffle",
+ "IntDomainFloatShuffle","true",
+ "Has bypass delay when using float shuffle type",
+ [], InlineIgnore>;
+
// Prefer lowering shuffles on AVX512 targets (e.g. Skylake Server) to
// imm shifts/rotate if they can use more ports than regular shuffles.
def TuningPreferShiftShuffle : SubtargetFeature<"faster-shift-than-shuffle",
@@ -1622,7 +1629,8 @@ def ProcessorFeatures {
TuningFastScalarShiftMasks,
TuningBranchFusion,
TuningSBBDepBreaking,
- TuningInsertVZEROUPPER];
+ TuningInsertVZEROUPPER,
+ TuningIntDomainFloatShuffle];
// PileDriver
list<SubtargetFeature> BdVer2AdditionalFeatures = [FeatureF16C,
@@ -1702,7 +1710,8 @@ def ProcessorFeatures {
TuningSlowSHLD,
TuningSBBDepBreaking,
TuningInsertVZEROUPPER,
- TuningAllowLight256Bit];
+ TuningAllowLight256Bit,
+ TuningIntDomainFloatShuffle];
list<SubtargetFeature> ZN2AdditionalFeatures = [FeatureCLWB,
FeatureRDPID,
FeatureRDPRU,
diff --git a/llvm/lib/Target/X86/X86FixupInstTuning.cpp b/llvm/lib/Target/X86/X86FixupInstTuning.cpp
index 621fd35a03cd5..6498464645e1f 100644
--- a/llvm/lib/Target/X86/X86FixupInstTuning.cpp
+++ b/llvm/lib/Target/X86/X86FixupInstTuning.cpp
@@ -138,6 +138,62 @@ bool X86FixupInstTuningImpl::processInstruction(
return ReplaceInTie;
};
+ // `movaps r1, r2` + `shufps r1, r2, i` -> `pshufd r1, r2, i`
+ // SHUFPS is destructive (2-address), so instruction selection may introduce
+ // a preceding copy to preserve the original source value. PSHUFD is
+ // non-destructive, making the copy unnecessary.
+ // We skip NewOpcPreferable here because we are explicitly replacing two
+ // instructions with one, which is universally profitable.
+ auto ProcessSHUFPS = [&]() -> bool {
+ if (!ST->hasSSE2() ||
+ (!ST->hasNoDomainDelayShuffle() && !ST->hasIntDomainFloatShuffle()))
+ return false;
+
+ if (I == MBB.begin())
+ return false;
+
+ MachineInstr &PrevMI = *std::prev(I);
+ unsigned PrevOpc = PrevMI.getOpcode();
+ if (PrevOpc != X86::MOVAPSrr && PrevOpc != X86::MOVAPDrr &&
+ PrevOpc != X86::MOVUPSrr && PrevOpc != X86::MOVUPDrr)
+ return false;
+
+ const MachineOperand &DstOp = MI.getOperand(0);
+ const MachineOperand &Src1Op = MI.getOperand(1);
+ const MachineOperand &Src2Op = MI.getOperand(2);
+
+ const MachineOperand &PrevDstOp = PrevMI.getOperand(0);
+ const MachineOperand &PrevSrcOp = PrevMI.getOperand(1);
+
+ const Register DstReg = DstOp.getReg();
+ const Register Src1Reg = Src1Op.getReg();
+ const Register Src2Reg = Src2Op.getReg();
+
+ if (DstReg != Src1Reg)
+ return false;
+
+ // Check that MOVAPS matches: `DstReg = MOVAPS Src2Reg`
+ if (PrevDstOp.getReg() != DstReg || PrevSrcOp.getReg() != Src2Reg)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "Replacing: " << PrevMI << " and " << MI);
+ unsigned MaskImm = MI.getOperand(NumOperands - 1).getImm();
+ bool SrcIsKill = Src2Op.isKill();
+
+ MachineInstr *NewMI =
+ BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(X86::PSHUFDri))
+ .add(DstOp) // Preserves dead flags etc.
+ .addReg(Src2Reg, getKillRegState(SrcIsKill))
+ .addImm(MaskImm);
+
+ LLVM_DEBUG(dbgs() << " With: " << *NewMI);
+
+ PrevMI.eraseFromParent();
+ I = MBB.erase(MI);
+ --I;
+ return true;
+ };
+
// `vpermilpd r, i` -> `vshufpd r, r, i`
// `vpermilpd r, i, k` -> `vshufpd r, r, i, k`
// `vshufpd` is always as fast or faster than `vpermilpd` and takes
@@ -698,6 +754,9 @@ bool X86FixupInstTuningImpl::processInstruction(
case X86::VPSLLQZri:
return ProcessShiftLeftToAdd(X86::VPADDQZrr);
+ case X86::SHUFPSrri:
+ return ProcessSHUFPS();
+
default:
return false;
}
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 6cfea56457910..ccb4516b59699 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -253,6 +253,7 @@ class X86Subtarget final : public X86GenSubtargetInfo {
bool hasNoDomainDelayShuffle() const {
return hasNoDomainDelay() || NoDomainDelayShuffle;
}
+ bool hasIntDomainFloatShuffle() const { return IntDomainFloatShuffle; }
// If there are no 512-bit vectors and we prefer not to use 512-bit registers,
// disable them in the legalizer.
diff --git a/llvm/test/CodeGen/X86/fixup-shufps-to-pshufd.ll b/llvm/test/CodeGen/X86/fixup-shufps-to-pshufd.ll
index e258285700418..794c2c705bbca 100644
--- a/llvm/test/CodeGen/X86/fixup-shufps-to-pshufd.ll
+++ b/llvm/test/CodeGen/X86/fixup-shufps-to-pshufd.ll
@@ -103,12 +103,12 @@ define <4 x float> @shuffle_v4f32_1032(<4 x float> %a) {
define <4 x float> @shuffle_v4f32_two_source(<4 x float> %a, <4 x float> %b) {
; SSE2-LABEL: shuffle_v4f32_two_source:
; SSE2: # %bb.0:
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE4-LABEL: shuffle_v4f32_two_source:
; SSE4: # %bb.0:
-; SSE4-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE4-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE4-NEXT: retq
;
; NO-DELAY-LABEL: shuffle_v4f32_two_source:
@@ -151,8 +151,7 @@ define void @shuffle_v4f32_with_liveout(<4 x float> %a, ptr %out1, ptr %out2) {
; NO-DELAY-LABEL: shuffle_v4f32_with_liveout:
; NO-DELAY: # %bb.0:
; NO-DELAY-NEXT: addps %xmm0, %xmm0
-; NO-DELAY-NEXT: movaps %xmm0, %xmm1
-; NO-DELAY-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,2],xmm0[2,2]
+; NO-DELAY-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,2,2]
; NO-DELAY-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; NO-DELAY-NEXT: movaps %xmm1, (%rdi)
; NO-DELAY-NEXT: movaps %xmm0, (%rsi)
@@ -161,8 +160,7 @@ define void @shuffle_v4f32_with_liveout(<4 x float> %a, ptr %out1, ptr %out2) {
; ZNVER2-LABEL: shuffle_v4f32_with_liveout:
; ZNVER2: # %bb.0:
; ZNVER2-NEXT: addps %xmm0, %xmm0
-; ZNVER2-NEXT: movaps %xmm0, %xmm1
-; ZNVER2-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,2],xmm0[2,2]
+; ZNVER2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,2,2]
; ZNVER2-NEXT: addps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; ZNVER2-NEXT: movaps %xmm1, (%rdi)
; ZNVER2-NEXT: movaps %xmm0, (%rsi)
More information about the llvm-commits
mailing list