[llvm] [DAG][X86] Avoid slow masked-store folds on Zen and x86-64-v3 (PR #214187)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 03:19:00 PDT 2026
https://github.com/222rohan updated https://github.com/llvm/llvm-project/pull/214187
>From 7d5e7252c8011618684e72a9094f3ffa5a7e3792 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <rshenoy at amd.com>
Date: Thu, 6 Aug 2026 15:35:01 +0530
Subject: [PATCH] [DAG][X86] Avoid slow masked-store folds on Zen and x86-64-v3
PR #145176 added foldToMaskedStore, which folds select + store
sequences into masked stores. Avoid that fold on CPUs where
vpmaskmov is slow, while preserving intrinsics.
Fixes #213195.
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 +
llvm/lib/Target/X86/X86.td | 14 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 4 +
llvm/test/CodeGen/X86/slow-vpmaskmov.ll | 215 ++++++++++++++++++
4 files changed, 233 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/slow-vpmaskmov.ll
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 06c9233d2f497..9f4b100461f8a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24322,7 +24322,10 @@ static SDValue foldToMaskedStore(StoreSDNode *Store, SelectionDAG &DAG,
Align Alignment = Store->getAlign();
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ // A legal masked store can still be slower than the original sequence,
+ // e.g. on pre-AVX-512 Zen, which we avoid by checking isTypeDesirableForOp.
if (!TLI.isOperationLegalOrCustom(ISD::MSTORE, VT) ||
+ !TLI.isTypeDesirableForOp(ISD::MSTORE, VT) ||
!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment))
return SDValue();
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index f40b937efa74b..5a36549a61bc4 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -514,6 +514,11 @@ def TuningSlowPMADDWD : SubtargetFeature<"slow-pmaddwd", "IsPMADDWDSlow",
"PMADDWD is slower than PMULLD",
[], InlineIgnore>;
+def TuningSlowVecMaskStore
+ : SubtargetFeature<"slow-vec-mask-store", "IsVecMaskStoreSlow", "true",
+ "Vector mask store instruction is slow",
+ [], InlineIgnore>;
+
// FIXME: This should not apply to CPUs that do not have SSE.
def TuningSlowUAMem16 : SubtargetFeature<"slow-unaligned-mem-16",
"IsUnalignedMem16Slow", "true",
@@ -990,7 +995,8 @@ def ProcessorFeatures {
TuningLZCNTFalseDeps,
TuningTZCNTFalseDeps,
TuningInsertVZEROUPPER,
- TuningAllowLight256Bit
+ TuningAllowLight256Bit,
+ TuningSlowVecMaskStore
];
list<SubtargetFeature> X86_64V4Features = !listconcat(X86_64V3Features, [
@@ -1695,7 +1701,8 @@ def ProcessorFeatures {
TuningSlowSHLD,
TuningSBBDepBreaking,
TuningInsertVZEROUPPER,
- TuningAllowLight256Bit];
+ TuningAllowLight256Bit,
+ TuningSlowVecMaskStore];
list<SubtargetFeature> ZN2AdditionalFeatures = [FeatureCLWB,
FeatureRDPID,
FeatureRDPRU,
@@ -1718,7 +1725,8 @@ def ProcessorFeatures {
TuningCOMPRESSFalseDeps,
TuningEXPANDFalseDeps];
list<SubtargetFeature> ZN4Tuning =
- !listconcat(ZN3Tuning, ZN4AdditionalTuning);
+ !listremove(!listconcat(ZN3Tuning, ZN4AdditionalTuning),
+ [TuningSlowVecMaskStore]);
list<SubtargetFeature> ZN4AdditionalFeatures = [FeatureAVX512,
FeatureCDI,
FeatureDQI,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index d78c478ab672f..de479e8f64096 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -63205,6 +63205,10 @@ bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
if (!isTypeLegal(VT))
return false;
+ // Legacy AVX/AVX2 masked stores are slow on pre-AVX-512 Zen CPUs.
+ if (Opc == ISD::MSTORE && Subtarget.isVecMaskStoreSlow())
+ return false;
+
// There are no vXi8 shifts.
if (Opc == ISD::SHL && VT.isVectorOf(MVT::i8))
return false;
diff --git a/llvm/test/CodeGen/X86/slow-vpmaskmov.ll b/llvm/test/CodeGen/X86/slow-vpmaskmov.ll
new file mode 100644
index 0000000000000..03c4d3b986fb5
--- /dev/null
+++ b/llvm/test/CodeGen/X86/slow-vpmaskmov.ll
@@ -0,0 +1,215 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=znver1 | FileCheck %s --check-prefixes=SLOW,SLOW-ZN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=SLOW,SLOW-V3
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell | FileCheck %s --check-prefix=AVX-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=znver4 | FileCheck %s --check-prefix=AVX512-FAST
+
+; Same-address load/select/store with zero must use ordinary vector memory
+; operations on pre-AVX-512 Zen.
+define void @load_select_zero(ptr %x, <8 x i1> %mask) {
+; SLOW-LABEL: load_select_zero:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SLOW-NEXT: vpslld $31, %ymm0, %ymm0
+; SLOW-NEXT: vpsrad $31, %ymm0, %ymm0
+; SLOW-NEXT: vpand (%rdi), %ymm0, %ymm0
+; SLOW-NEXT: vmovdqu %ymm0, (%rdi)
+; SLOW-NEXT: vzeroupper
+; SLOW-NEXT: retq
+;
+; AVX-FAST-LABEL: load_select_zero:
+; AVX-FAST: # %bb.0:
+; AVX-FAST-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-FAST-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX-FAST-NEXT: vpxor %xmm2, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX-FAST-NEXT: vpslld $31, %ymm0, %ymm0
+; AVX-FAST-NEXT: vpmaskmovd %ymm1, %ymm0, (%rdi)
+; AVX-FAST-NEXT: vzeroupper
+; AVX-FAST-NEXT: retq
+;
+; AVX512-FAST-LABEL: load_select_zero:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpsllw $15, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vpmovw2m %xmm0, %k0
+; AVX512-FAST-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-FAST-NEXT: knotb %k0, %k1
+; AVX512-FAST-NEXT: vmovdqu32 %ymm0, (%rdi) {%k1}
+; AVX512-FAST-NEXT: vzeroupper
+; AVX512-FAST-NEXT: retq
+ %load = load <8 x i32>, ptr %x, align 4
+ %select = select <8 x i1> %mask, <8 x i32> %load, <8 x i32> zeroinitializer
+ store <8 x i32> %select, ptr %x, align 4
+ ret void
+}
+
+; Cover the other vselect operand order.
+define void @select_value_load_v4i64(ptr %x, <4 x i64> %value, <4 x i1> %mask) {
+; SLOW-ZN-LABEL: select_value_load_v4i64:
+; SLOW-ZN: # %bb.0:
+; SLOW-ZN-NEXT: vmovupd (%rdi), %ymm2
+; SLOW-ZN-NEXT: vpslld $31, %xmm1, %xmm1
+; SLOW-ZN-NEXT: vpmovsxdq %xmm1, %ymm1
+; SLOW-ZN-NEXT: vblendvpd %ymm1, %ymm0, %ymm2, %ymm0
+; SLOW-ZN-NEXT: vmovupd %ymm0, (%rdi)
+; SLOW-ZN-NEXT: vzeroupper
+; SLOW-ZN-NEXT: retq
+;
+; SLOW-V3-LABEL: select_value_load_v4i64:
+; SLOW-V3: # %bb.0:
+; SLOW-V3-NEXT: vpslld $31, %xmm1, %xmm1
+; SLOW-V3-NEXT: vpmovsxdq %xmm1, %ymm1
+; SLOW-V3-NEXT: vmovupd (%rdi), %ymm2
+; SLOW-V3-NEXT: vblendvpd %ymm1, %ymm0, %ymm2, %ymm0
+; SLOW-V3-NEXT: vmovupd %ymm0, (%rdi)
+; SLOW-V3-NEXT: vzeroupper
+; SLOW-V3-NEXT: retq
+;
+; AVX-FAST-LABEL: select_value_load_v4i64:
+; AVX-FAST: # %bb.0:
+; AVX-FAST-NEXT: vpslld $31, %xmm1, %xmm1
+; AVX-FAST-NEXT: vpmovsxdq %xmm1, %ymm1
+; AVX-FAST-NEXT: vpmaskmovq %ymm0, %ymm1, (%rdi)
+; AVX-FAST-NEXT: vzeroupper
+; AVX-FAST-NEXT: retq
+;
+; AVX512-FAST-LABEL: select_value_load_v4i64:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpslld $31, %xmm1, %xmm1
+; AVX512-FAST-NEXT: vpmovd2m %xmm1, %k1
+; AVX512-FAST-NEXT: vmovdqu64 %ymm0, (%rdi) {%k1}
+; AVX512-FAST-NEXT: vzeroupper
+; AVX512-FAST-NEXT: retq
+ %load = load <4 x i64>, ptr %x, align 4
+ %select = select <4 x i1> %mask, <4 x i64> %value, <4 x i64> %load
+ store <4 x i64> %select, ptr %x, align 4
+ ret void
+}
+
+; Do not change explicit masked stores.
+define void @explicit_masked_store(ptr %x, <8 x i32> %value, <8 x i1> %mask) {
+; SLOW-LABEL: explicit_masked_store:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SLOW-NEXT: vpslld $31, %ymm1, %ymm1
+; SLOW-NEXT: vpmaskmovd %ymm0, %ymm1, (%rdi)
+; SLOW-NEXT: vzeroupper
+; SLOW-NEXT: retq
+;
+; AVX-FAST-LABEL: explicit_masked_store:
+; AVX-FAST: # %bb.0:
+; AVX-FAST-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX-FAST-NEXT: vpslld $31, %ymm1, %ymm1
+; AVX-FAST-NEXT: vpmaskmovd %ymm0, %ymm1, (%rdi)
+; AVX-FAST-NEXT: vzeroupper
+; AVX-FAST-NEXT: retq
+;
+; AVX512-FAST-LABEL: explicit_masked_store:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpsllw $15, %xmm1, %xmm1
+; AVX512-FAST-NEXT: vpmovw2m %xmm1, %k1
+; AVX512-FAST-NEXT: vmovdqu32 %ymm0, (%rdi) {%k1}
+; AVX512-FAST-NEXT: vzeroupper
+; AVX512-FAST-NEXT: retq
+ call void @llvm.masked.store.v8i32.p0(<8 x i32> %value, ptr %x, i32 4,
+ <8 x i1> %mask)
+ ret void
+}
+
+define void @load_select_zero_v4f32(ptr %x, <4 x i1> %mask) {
+; SLOW-LABEL: load_select_zero_v4f32:
+; SLOW: # %bb.0:
+; SLOW-NEXT: vpslld $31, %xmm0, %xmm0
+; SLOW-NEXT: vpsrad $31, %xmm0, %xmm0
+; SLOW-NEXT: vpand (%rdi), %xmm0, %xmm0
+; SLOW-NEXT: vmovdqu %xmm0, (%rdi)
+; SLOW-NEXT: retq
+;
+; AVX-FAST-LABEL: load_select_zero_v4f32:
+; AVX-FAST: # %bb.0:
+; AVX-FAST-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX-FAST-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX-FAST-NEXT: vpxor %xmm2, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpslld $31, %xmm0, %xmm0
+; AVX-FAST-NEXT: vmaskmovps %xmm1, %xmm0, (%rdi)
+; AVX-FAST-NEXT: retq
+;
+; AVX512-FAST-LABEL: load_select_zero_v4f32:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpslld $31, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vpmovd2m %xmm0, %k0
+; AVX512-FAST-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-FAST-NEXT: knotw %k0, %k1
+; AVX512-FAST-NEXT: vmovups %xmm0, (%rdi) {%k1}
+; AVX512-FAST-NEXT: retq
+ %load = load <4 x float>, ptr %x, align 4
+ %select = select <4 x i1> %mask, <4 x float> %load,
+ <4 x float> zeroinitializer
+ store <4 x float> %select, ptr %x, align 4
+ ret void
+}
+
+define void @load_select_zero_v16i32(ptr %x, <16 x i1> %mask) {
+; SLOW-ZN-LABEL: load_select_zero_v16i32:
+; SLOW-ZN: # %bb.0:
+; SLOW-ZN-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SLOW-ZN-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; SLOW-ZN-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SLOW-ZN-NEXT: vpslld $31, %ymm0, %ymm0
+; SLOW-ZN-NEXT: vpslld $31, %ymm1, %ymm1
+; SLOW-ZN-NEXT: vpsrad $31, %ymm0, %ymm0
+; SLOW-ZN-NEXT: vpand (%rdi), %ymm0, %ymm0
+; SLOW-ZN-NEXT: vpsrad $31, %ymm1, %ymm1
+; SLOW-ZN-NEXT: vpand 32(%rdi), %ymm1, %ymm1
+; SLOW-ZN-NEXT: vmovdqu %ymm0, (%rdi)
+; SLOW-ZN-NEXT: vmovdqu %ymm1, 32(%rdi)
+; SLOW-ZN-NEXT: vzeroupper
+; SLOW-ZN-NEXT: retq
+;
+; SLOW-V3-LABEL: load_select_zero_v16i32:
+; SLOW-V3: # %bb.0:
+; SLOW-V3-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SLOW-V3-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SLOW-V3-NEXT: vpslld $31, %ymm1, %ymm1
+; SLOW-V3-NEXT: vpsrad $31, %ymm1, %ymm1
+; SLOW-V3-NEXT: vpand 32(%rdi), %ymm1, %ymm1
+; SLOW-V3-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; SLOW-V3-NEXT: vpslld $31, %ymm0, %ymm0
+; SLOW-V3-NEXT: vpsrad $31, %ymm0, %ymm0
+; SLOW-V3-NEXT: vpand (%rdi), %ymm0, %ymm0
+; SLOW-V3-NEXT: vmovdqu %ymm0, (%rdi)
+; SLOW-V3-NEXT: vmovdqu %ymm1, 32(%rdi)
+; SLOW-V3-NEXT: vzeroupper
+; SLOW-V3-NEXT: retq
+;
+; AVX-FAST-LABEL: load_select_zero_v16i32:
+; AVX-FAST: # %bb.0:
+; AVX-FAST-NEXT: vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX-FAST-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX-FAST-NEXT: vpxor %ymm3, %ymm1, %ymm1
+; AVX-FAST-NEXT: vpslld $31, %ymm1, %ymm1
+; AVX-FAST-NEXT: vpmaskmovd %ymm2, %ymm1, (%rdi)
+; AVX-FAST-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; AVX-FAST-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX-FAST-NEXT: vpxor %ymm3, %ymm0, %ymm0
+; AVX-FAST-NEXT: vpslld $31, %ymm0, %ymm0
+; AVX-FAST-NEXT: vpmaskmovd %ymm2, %ymm0, 32(%rdi)
+; AVX-FAST-NEXT: vzeroupper
+; AVX-FAST-NEXT: retq
+;
+; AVX512-FAST-LABEL: load_select_zero_v16i32:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpsllw $7, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vpmovb2m %xmm0, %k0
+; AVX512-FAST-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-FAST-NEXT: knotw %k0, %k1
+; AVX512-FAST-NEXT: vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512-FAST-NEXT: vzeroupper
+; AVX512-FAST-NEXT: retq
+ %load = load <16 x i32>, ptr %x, align 4
+ %select = select <16 x i1> %mask, <16 x i32> %load,
+ <16 x i32> zeroinitializer
+ store <16 x i32> %select, ptr %x, align 4
+ ret void
+}
More information about the llvm-commits
mailing list