[llvm] [DAG][X86] Avoid slow masked-store folds on Zen (PR #214187)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 5 03:20:15 PDT 2026
https://github.com/222rohan created https://github.com/llvm/llvm-project/pull/214187
PR #145176 added a method (`foldToMaskedStore`) which folds same-address load/select/store sequences into `masked_store`. On non-AVX-512 Zen CPUs, this may select the slow legacy `vpmaskmovd/q` store instead of normal vector memory operations.
Add a TargetLowering profitability hook. It accepts the fold by default, while X86 rejects it for CPUs with the new slow mask-move tuning (added to Zen targets); explicit masked-store intrinsics and AVX-512 lowering are unchanged.
Addresses #213195 for Zen.
mca: https://godbolt.org/z/7brjMzj15
New `slow-vpmaskmov.ll` test added.
>From 678d04185471b2b1f9eec118fee2fe67e1bf936a Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <rshenoy at amd.com>
Date: Tue, 4 Aug 2026 16:47:42 +0530
Subject: [PATCH] [DAG][X86] Avoid slow masked-store folds on Zen
PR #145176 added foldToMaskedStore, which folds select + store
sequences into masked stores. Avoid that fold on Zen CPUs where
legacy vpmaskmov is slow, while preserving intrinsics and AVX-512
lowering path.
Addresses #213195 for Zen subtargets.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 5 +
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 1 +
llvm/lib/Target/X86/X86.td | 8 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 4 +
llvm/lib/Target/X86/X86ISelLowering.h | 2 +
llvm/test/CodeGen/X86/slow-vpmaskmov.ll | 109 ++++++++++++++++++
6 files changed, 128 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/X86/slow-vpmaskmov.ll
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index c663bb8ea65b7..6c2b3e9582bd1 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4622,6 +4622,11 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
// For targets which wrap address, unwrap for analysis.
virtual SDValue unwrapAddress(SDValue N) const { return N; }
+ /// Return true if it is profitable to fold a same-address vector
+ /// load-select-store sequence into a masked store. The default preserves the
+ /// existing combine.
+ virtual bool shouldFoldStoreToMaskedStore(EVT /*VT*/) const { return true; }
+
/// Returns true (and the GlobalValue and the offset) if the node is a
/// GlobalAddress + offset.
virtual bool
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 06c9233d2f497..5112384d29d59 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24323,6 +24323,7 @@ static SDValue foldToMaskedStore(StoreSDNode *Store, SelectionDAG &DAG,
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
if (!TLI.isOperationLegalOrCustom(ISD::MSTORE, VT) ||
+ !TLI.shouldFoldStoreToMaskedStore(VT) ||
!TLI.allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment))
return SDValue();
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index f40b937efa74b..a26cc1f079429 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -514,6 +514,11 @@ def TuningSlowPMADDWD : SubtargetFeature<"slow-pmaddwd", "IsPMADDWDSlow",
"PMADDWD is slower than PMULLD",
[], InlineIgnore>;
+def TuningSlowVecMaskMove
+ : SubtargetFeature<"slow-vec-maskmove", "IsVecMaskMoveSlow", "true",
+ "Vector mask move instruction is slow",
+ [], InlineIgnore>;
+
// FIXME: This should not apply to CPUs that do not have SSE.
def TuningSlowUAMem16 : SubtargetFeature<"slow-unaligned-mem-16",
"IsUnalignedMem16Slow", "true",
@@ -1695,7 +1700,8 @@ def ProcessorFeatures {
TuningSlowSHLD,
TuningSBBDepBreaking,
TuningInsertVZEROUPPER,
- TuningAllowLight256Bit];
+ TuningAllowLight256Bit,
+ TuningSlowVecMaskMove];
list<SubtargetFeature> ZN2AdditionalFeatures = [FeatureCLWB,
FeatureRDPID,
FeatureRDPRU,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index d78c478ab672f..350e834d0153a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -2883,6 +2883,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
IsStrictFPEnabled = true;
}
+bool X86TargetLowering::shouldFoldStoreToMaskedStore(EVT) const {
+ return !Subtarget.isVecMaskMoveSlow() || Subtarget.hasAVX512();
+}
+
// This has so far only been implemented for 64-bit MachO.
bool X86TargetLowering::useLoadStackGuardNode(const Module &M) const {
return Subtarget.isTargetMachO() && Subtarget.is64Bit();
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 798050028c15a..a73a600d9f1e7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -184,6 +184,8 @@ namespace llvm {
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
+ bool shouldFoldStoreToMaskedStore(EVT VT) const override;
+
bool preferABDSToABSWithNSW(EVT VT) const override;
bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT,
diff --git a/llvm/test/CodeGen/X86/slow-vpmaskmov.ll b/llvm/test/CodeGen/X86/slow-vpmaskmov.ll
new file mode 100644
index 0000000000000..1cf77ff8b16e9
--- /dev/null
+++ b/llvm/test/CodeGen/X86/slow-vpmaskmov.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=znver1 | FileCheck %s --check-prefix=ZEN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=znver5 -mattr=-avx512f | FileCheck %s --check-prefix=ZEN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell | FileCheck %s --check-prefix=LEGACY
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=sapphirerapids | FileCheck %s --check-prefix=AVX512
+
+declare void @llvm.masked.store.v8i32.p0(<8 x i32>, ptr, i32, <8 x i1>)
+
+; Same-address load/select/store with zero must use ordinary vector memory
+; operations on pre-AVX-512 Zen.
+define void @load_select_zero(ptr %x, <8 x i1> %mask) {
+; ZEN-LABEL: load_select_zero:
+; ZEN: # %bb.0:
+; ZEN-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; ZEN-NEXT: vpslld $31, %ymm0, %ymm0
+; ZEN-NEXT: vpsrad $31, %ymm0, %ymm0
+; ZEN-NEXT: vpand (%rdi), %ymm0, %ymm0
+; ZEN-NEXT: vmovdqu %ymm0, (%rdi)
+; ZEN-NEXT: vzeroupper
+; ZEN-NEXT: retq
+;
+; LEGACY-LABEL: load_select_zero:
+; LEGACY: # %bb.0:
+; LEGACY-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; LEGACY-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
+; LEGACY-NEXT: vpxor %xmm2, %xmm0, %xmm0
+; LEGACY-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; LEGACY-NEXT: vpslld $31, %ymm0, %ymm0
+; LEGACY-NEXT: vpmaskmovd %ymm1, %ymm0, (%rdi)
+; LEGACY-NEXT: vzeroupper
+; LEGACY-NEXT: retq
+;
+; AVX512-LABEL: load_select_zero:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpsllw $15, %xmm0, %xmm0
+; AVX512-NEXT: vpmovw2m %xmm0, %k0
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: knotb %k0, %k1
+; AVX512-NEXT: vmovdqu32 %ymm0, (%rdi) {%k1}
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %load = load <8 x i32>, ptr %x, align 4
+ %select = select <8 x i1> %mask, <8 x i32> %load, <8 x i32> zeroinitializer
+ store <8 x i32> %select, ptr %x, align 4
+ ret void
+}
+
+; Cover the other vselect operand order.
+define void @select_value_load(ptr %x, <8 x i32> %value, <8 x i1> %mask) {
+; ZEN-LABEL: select_value_load:
+; ZEN: # %bb.0:
+; ZEN-NEXT: vmovups (%rdi), %ymm2
+; ZEN-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; ZEN-NEXT: vpslld $31, %ymm1, %ymm1
+; ZEN-NEXT: vblendvps %ymm1, %ymm0, %ymm2, %ymm0
+; ZEN-NEXT: vmovups %ymm0, (%rdi)
+; ZEN-NEXT: vzeroupper
+; ZEN-NEXT: retq
+;
+; LEGACY-LABEL: select_value_load:
+; LEGACY: # %bb.0:
+; LEGACY-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; LEGACY-NEXT: vpslld $31, %ymm1, %ymm1
+; LEGACY-NEXT: vpmaskmovd %ymm0, %ymm1, (%rdi)
+; LEGACY-NEXT: vzeroupper
+; LEGACY-NEXT: retq
+;
+; AVX512-LABEL: select_value_load:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1
+; AVX512-NEXT: vpmovw2m %xmm1, %k1
+; AVX512-NEXT: vmovdqu32 %ymm0, (%rdi) {%k1}
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %load = load <8 x i32>, ptr %x, align 4
+ %select = select <8 x i1> %mask, <8 x i32> %value, <8 x i32> %load
+ store <8 x i32> %select, ptr %x, align 4
+ ret void
+}
+
+; Explicit masked stores keep their fault-suppression semantics.
+define void @explicit_masked_store(ptr %x, <8 x i32> %value, <8 x i1> %mask) {
+; ZEN-LABEL: explicit_masked_store:
+; ZEN: # %bb.0:
+; ZEN-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; ZEN-NEXT: vpslld $31, %ymm1, %ymm1
+; ZEN-NEXT: vpmaskmovd %ymm0, %ymm1, (%rdi)
+; ZEN-NEXT: vzeroupper
+; ZEN-NEXT: retq
+;
+; LEGACY-LABEL: explicit_masked_store:
+; LEGACY: # %bb.0:
+; LEGACY-NEXT: vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; LEGACY-NEXT: vpslld $31, %ymm1, %ymm1
+; LEGACY-NEXT: vpmaskmovd %ymm0, %ymm1, (%rdi)
+; LEGACY-NEXT: vzeroupper
+; LEGACY-NEXT: retq
+;
+; AVX512-LABEL: explicit_masked_store:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpsllw $15, %xmm1, %xmm1
+; AVX512-NEXT: vpmovw2m %xmm1, %k1
+; AVX512-NEXT: vmovdqu32 %ymm0, (%rdi) {%k1}
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ call void @llvm.masked.store.v8i32.p0(<8 x i32> %value, ptr %x, i32 4,
+ <8 x i1> %mask)
+ ret void
+}
More information about the llvm-commits
mailing list