[llvm] [LoopVectorize][X86] Rewrite guaranteed-safe masked stores as load-blend-store] (PR #206654)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 29 23:37:34 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Arjun H Kumar (arjun-harikumar-amd)
<details>
<summary>Changes</summary>
---
Patch is 58.02 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/206654.diff
26 Files Affected:
- (added) llvm/include/llvm/Analysis/MemSafetyAnalysis.h (+121)
- (modified) llvm/include/llvm/Analysis/TargetTransformInfo.h (+8)
- (modified) llvm/include/llvm/Analysis/TargetTransformInfoImpl.h (+4)
- (modified) llvm/lib/Analysis/CMakeLists.txt (+1)
- (added) llvm/lib/Analysis/MemSafetyAnalysis.cpp (+237)
- (modified) llvm/lib/Analysis/TargetTransformInfo.cpp (+4)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h (+11)
- (modified) llvm/lib/Target/X86/X86.td (+12-1)
- (modified) llvm/lib/Target/X86/X86TargetTransformInfo.cpp (+8)
- (modified) llvm/lib/Target/X86/X86TargetTransformInfo.h (+2)
- (modified) llvm/lib/Transforms/Vectorize/LoopVectorize.cpp (+135-2)
- (modified) llvm/lib/Transforms/Vectorize/VPlan.h (+23-3)
- (modified) llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp (+31-3)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/optimize-masked-memory/neon-flag-off.ll (+41)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/optimize-masked-memory/neon-guaranteed-safe.ll (+50)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/optimize-masked-memory/neon-non-guaranteed.ll (+41)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/optimize-masked-memory/neon-scatter-untouched.ll (+40)
- (added) llvm/test/Transforms/LoopVectorize/AArch64/optimize-masked-memory/sve-neutral.ll (+43)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/predication_costs.ll (+1-1)
- (modified) llvm/test/Transforms/LoopVectorize/AArch64/transform-narrow-interleave-to-widen-memory-multi-block.ll (+2-2)
- (added) llvm/test/Transforms/LoopVectorize/Inputs/check-first-masked-replay-bench.c (+103)
- (modified) llvm/test/Transforms/LoopVectorize/VPlan/AArch64/single-scalar-cast.ll (+1-1)
- (added) llvm/test/Transforms/LoopVectorize/X86/optimize-masked-memory/mode-a-flag-off.ll (+42)
- (added) llvm/test/Transforms/LoopVectorize/X86/optimize-masked-memory/mode-a-guaranteed-safe.ll (+44)
- (added) llvm/test/Transforms/LoopVectorize/X86/optimize-masked-memory/mode-a-non-guaranteed.ll (+40)
- (added) llvm/test/Transforms/LoopVectorize/X86/optimize-masked-memory/scatter-untouched.ll (+42)
``````````diff
diff --git a/llvm/include/llvm/Analysis/MemSafetyAnalysis.h b/llvm/include/llvm/Analysis/MemSafetyAnalysis.h
new file mode 100644
index 0000000000000..baa47f70a6d39
--- /dev/null
+++ b/llvm/include/llvm/Analysis/MemSafetyAnalysis.h
@@ -0,0 +1,121 @@
+//===- MemSafetyAnalysis.h - Memory access safety for a loop ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Originally developed by Advanced Micro Devices, Inc. (2015).
+//
+// MemSafetyAnalysis answers one question about a loop's memory accesses:
+//
+// "Is this memory access guaranteed to execute on every iteration of the
+// loop?" -- isGuaranteedMemoryAccess(const SCEV *)
+//
+// Used to legalise rewriting a masked store as load-blend-store: if the
+// same SCEV-keyed address is touched on every iteration anyway, the
+// sequence does not introduce any new fault.
+//
+// The analysis is built once per loop and consulted from the loop vectoriser
+// when -enable-masked-memory-optimization is in effect.
+//
+// Phase-2 (stack-alloca padding) is layered on top of this analysis in a
+// follow-up patch; the ctor here keeps the TLI/TTI parameters that Phase-2
+// will use, so the analysis-construction call site does not have to change.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_MEMSAFETYANALYSIS_H
+#define LLVM_ANALYSIS_MEMSAFETYANALYSIS_H
+
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/IR/Value.h"
+#include <map>
+
+namespace llvm {
+
+class BasicBlock;
+class DominatorTree;
+class Loop;
+class LoopInfo;
+class ScalarEvolution;
+class SCEV;
+class TargetLibraryInfo;
+class TargetTransformInfo;
+
+/// Classification of a per-block memory access. \c LocalSafe means the access
+/// has been observed but not yet proven to execute on every iteration of the
+/// enclosing loop; \c Safe means the access is proven to execute on every
+/// iteration; \c Unsafe is reserved for future use.
+enum class MemProperty { Unsafe, Safe, LocalSafe };
+
+/// Per-block summary of memory accesses, indexed by the SCEV of the pointer.
+class BlockMemInfo {
+public:
+ BasicBlock *BB;
+ bool GuaranteedToExecute;
+ std::map<const SCEV *, MemProperty> BlockMemAccess;
+
+ BlockMemInfo(BasicBlock *BB, bool GuaranteedToExecute)
+ : BB(BB), GuaranteedToExecute(GuaranteedToExecute) {}
+
+ void addMemoryAccess(const SCEV *S, MemProperty AccessTy);
+ bool isBlockGuaranteedToExecute() const { return GuaranteedToExecute; }
+ void copyBlockMemInfo(BlockMemInfo *BMI);
+};
+
+/// MemSafetyAnalysis -- per-loop analysis that builds a SCEV-keyed map of
+/// memory accesses guaranteed to execute on every iteration.
+///
+/// A memory access is guaranteed to execute when either:
+/// * it lives in a block that dominates the latch (top-level block), or
+/// * the same SCEV-keyed address is touched on every path of the
+/// subtree rooted at the current block (propagated bottom-up).
+///
+/// See `examples` block in MemSafetyAnalysis.cpp for canonical cases.
+class MemSafetyAnalysis {
+public:
+ MemSafetyAnalysis(Loop *L, LoopInfo *LI, ScalarEvolution *SE,
+ DominatorTree *DT, const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI);
+ ~MemSafetyAnalysis();
+
+ MemSafetyAnalysis(const MemSafetyAnalysis &) = delete;
+ MemSafetyAnalysis &operator=(const MemSafetyAnalysis &) = delete;
+
+ /// True if the analysis successfully ran on the loop. Clients must check
+ /// this before consulting the other queries.
+ bool isLegalAnalysis() const { return IsAnalysisValid; }
+
+ /// True if the access at SCEV \p S is guaranteed to execute on every
+ /// iteration of the loop body.
+ bool isGuaranteedMemoryAccess(const SCEV *S) const;
+
+ /// Debug: dump the analysis state to dbgs().
+ void printAnalysis() const;
+
+private:
+ Loop *L;
+ LoopInfo *LI;
+ ScalarEvolution *SE;
+ DominatorTree *DT;
+ const TargetLibraryInfo *TLI;
+ const TargetTransformInfo *TTI;
+ bool IsAnalysisValid = false;
+
+ std::map<BasicBlock *, BlockMemInfo *> BlockMemAccessMap;
+ std::map<const SCEV *, MemProperty> SafeMemoryAccesses;
+
+ BlockMemInfo *getBlockMemInfo(BasicBlock *BB);
+ bool blockGuaranteedToExecute(BasicBlock *BB);
+ void addGuaranteedMemoryAccess(const SCEV *S);
+ bool isLegalLoopStructure();
+ bool processBlock(BasicBlock *BB);
+ bool analyzeMemoryAccessesInLoop();
+ void clearLocalMemory();
+};
+
+} // namespace llvm
+
+#endif // LLVM_ANALYSIS_MEMSAFETYANALYSIS_H
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 10c0509460b95..bacf4545966b5 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1163,6 +1163,14 @@ class TargetTransformInfo {
/// vector loads/stores.
LLVM_ABI bool enableMaskedInterleavedAccessVectorization() const;
+ /// Return true if a masked store whose pointer is provably touched on every
+ /// loop iteration should be rewritten as an unconditional load + select +
+ /// unconditional store. Targets with a microcoded / slow masked-store path
+ /// (e.g. AMD znver1/2/3) should return true. Returning false leaves the
+ /// masked store untouched. Consulted by LoopVectorize when
+ /// -enable-masked-memory-optimization is on.
+ LLVM_ABI bool shouldRewriteMaskedStoreAsLoadBlendStore() const;
+
/// Indicate that it is potentially unsafe to automatically vectorize
/// floating-point operations because the semantics of vector and scalar
/// floating-point semantics may differ. For example, ARM NEON v7 SIMD math
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 5645cc63a6944..a391f23569757 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -553,6 +553,10 @@ class LLVM_ABI TargetTransformInfoImplBase {
return false;
}
+ virtual bool shouldRewriteMaskedStoreAsLoadBlendStore() const {
+ return false;
+ }
+
virtual bool isFPVectorizationPotentiallyUnsafe() const { return false; }
virtual bool allowsMisalignedMemoryAccesses(LLVMContext &Context,
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index f3586c66cb056..47f9a14180ad9 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -115,6 +115,7 @@ add_llvm_component_library(LLVMAnalysis
LoopPass.cpp
MLInlineAdvisor.cpp
MemDerefPrinter.cpp
+ MemSafetyAnalysis.cpp
MemoryBuiltins.cpp
MemoryDependenceAnalysis.cpp
MemoryLocation.cpp
diff --git a/llvm/lib/Analysis/MemSafetyAnalysis.cpp b/llvm/lib/Analysis/MemSafetyAnalysis.cpp
new file mode 100644
index 0000000000000..3b86203c81723
--- /dev/null
+++ b/llvm/lib/Analysis/MemSafetyAnalysis.cpp
@@ -0,0 +1,237 @@
+//===- MemSafetyAnalysis.cpp - Memory access safety for a loop ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Originally developed by Advanced Micro Devices, Inc. (2015).
+//
+// MemSafetyAnalysis identifies memory accesses that are guaranteed to execute
+// on every dynamic iteration of the loop body.
+//
+// Guaranteed-access classification:
+// Case#1: P is accessed in a top-level block (one that dominates the latch)
+// loop {
+// if (C1) *P;
+// else { if (C2) *P; else ... }
+// *P; // <-- top-level
+// }
+//
+// Case#2: P is accessed on every successor path of a top-level subtree
+// loop {
+// if (C1) *P;
+// else { if (C2) *P; else *P; } // both branches touch *P
+// }
+//
+// Case#3: P is not accessed on some path -- not guaranteed
+// loop {
+// if (C1) *P;
+// else { if (C2) *P; else { if (C3) *P; else /* no access */ } }
+// }
+//
+// See MemSafetyAnalysis.h for the public API.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/MemSafetyAnalysis.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopIterator.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include <stack>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "mem-access-safety"
+
+//===----------------------------------------------------------------------===//
+// BlockMemInfo
+//===----------------------------------------------------------------------===//
+
+void BlockMemInfo::addMemoryAccess(const SCEV *S, MemProperty AccessTy) {
+ BlockMemAccess[S] = AccessTy;
+}
+
+void BlockMemInfo::copyBlockMemInfo(BlockMemInfo *BMI) {
+ for (auto &Itr : BMI->BlockMemAccess)
+ BlockMemAccess[Itr.first] = Itr.second;
+}
+
+//===----------------------------------------------------------------------===//
+// MemSafetyAnalysis
+//===----------------------------------------------------------------------===//
+
+MemSafetyAnalysis::MemSafetyAnalysis(Loop *L, LoopInfo *LI, ScalarEvolution *SE,
+ DominatorTree *DT,
+ const TargetLibraryInfo *TLI,
+ const TargetTransformInfo *TTI)
+ : L(L), LI(LI), SE(SE), DT(DT), TLI(TLI), TTI(TTI) {
+ if (isLegalLoopStructure())
+ analyzeMemoryAccessesInLoop();
+}
+
+MemSafetyAnalysis::~MemSafetyAnalysis() {
+ // BlockMemAccessMap is cleared as part of analyzeMemoryAccessesInLoop's
+ // tail; no-op here if it already ran.
+ clearLocalMemory();
+}
+
+bool MemSafetyAnalysis::isLegalLoopStructure() {
+ if (!L->getLoopLatch())
+ return false;
+ if (!L->getSubLoops().empty())
+ return false;
+ if (L->getNumBackEdges() != 1)
+ return false;
+ if (!L->getExitingBlock())
+ return false;
+ if (L->getExitingBlock() != L->getLoopLatch())
+ return false;
+ return true;
+}
+
+void MemSafetyAnalysis::clearLocalMemory() {
+ for (auto &Itr : BlockMemAccessMap)
+ delete Itr.second;
+ BlockMemAccessMap.clear();
+}
+
+void MemSafetyAnalysis::addGuaranteedMemoryAccess(const SCEV *S) {
+ SafeMemoryAccesses[S] = MemProperty::Safe;
+}
+
+bool MemSafetyAnalysis::isGuaranteedMemoryAccess(const SCEV *S) const {
+ auto Itr = SafeMemoryAccesses.find(S);
+ if (Itr == SafeMemoryAccesses.end() || Itr->second != MemProperty::Safe)
+ return false;
+ return true;
+}
+
+bool MemSafetyAnalysis::blockGuaranteedToExecute(BasicBlock *BB) {
+ return DT->dominates(BB, L->getLoopLatch());
+}
+
+BlockMemInfo *MemSafetyAnalysis::getBlockMemInfo(BasicBlock *BB) {
+ auto Itr = BlockMemAccessMap.find(BB);
+ if (Itr != BlockMemAccessMap.end())
+ return Itr->second;
+ auto *BMI = new BlockMemInfo(BB, blockGuaranteedToExecute(BB));
+ BlockMemAccessMap[BB] = BMI;
+ return BMI;
+}
+
+bool MemSafetyAnalysis::processBlock(BasicBlock *BB) {
+ Instruction *Term = BB->getTerminator();
+ BlockMemInfo *BMI = getBlockMemInfo(BB);
+
+ if (auto *CB = dyn_cast<CondBrInst>(Term)) {
+ BlockMemInfo *Succ0BMI = getBlockMemInfo(CB->getSuccessor(0));
+ BlockMemInfo *Succ1BMI = getBlockMemInfo(CB->getSuccessor(1));
+ // Case#2: an access registered as LocalSafe on both successors gets
+ // promoted to Safe on the current block, and to a loop-level guaranteed
+ // access if the current block is itself guaranteed.
+ for (auto &Itr0 : Succ0BMI->BlockMemAccess) {
+ for (auto &Itr1 : Succ1BMI->BlockMemAccess) {
+ if (Itr0.first != Itr1.first)
+ continue;
+ if (Itr0.second != MemProperty::LocalSafe ||
+ Itr1.second != MemProperty::LocalSafe)
+ continue;
+ if (BMI->isBlockGuaranteedToExecute()) {
+ BMI->addMemoryAccess(Itr0.first, MemProperty::Safe);
+ addGuaranteedMemoryAccess(Itr0.first);
+ } else {
+ BMI->addMemoryAccess(Itr0.first, MemProperty::LocalSafe);
+ }
+ }
+ }
+ } else if (auto *UB = dyn_cast<UncondBrInst>(Term)) {
+ BlockMemInfo *Succ0BMI = getBlockMemInfo(UB->getSuccessor(0));
+ BMI->copyBlockMemInfo(Succ0BMI);
+ } else {
+ // Not a branch terminator (return, unreachable, switch, indirectbr,
+ // resume, ...) -- bail out and let the analysis stay invalid.
+ return false;
+ }
+
+ // Now register the block's own accesses.
+ for (auto &I : *BB) {
+ Value *Ptr = nullptr;
+ if (auto *Ld = dyn_cast<LoadInst>(&I)) {
+ if (!Ld->isSimple())
+ continue;
+ Ptr = Ld->getPointerOperand();
+ } else if (auto *St = dyn_cast<StoreInst>(&I)) {
+ if (!St->isSimple())
+ continue;
+ Ptr = St->getPointerOperand();
+ } else {
+ continue;
+ }
+
+ const SCEV *PtrSCEV = SE->getSCEV(Ptr);
+ if (isGuaranteedMemoryAccess(PtrSCEV)) {
+ BMI->addMemoryAccess(PtrSCEV, MemProperty::Safe);
+ continue;
+ }
+ if (BMI->isBlockGuaranteedToExecute()) {
+ addGuaranteedMemoryAccess(PtrSCEV);
+ BMI->addMemoryAccess(PtrSCEV, MemProperty::Safe);
+ continue;
+ }
+ BMI->addMemoryAccess(PtrSCEV, MemProperty::LocalSafe);
+ }
+ return true;
+}
+
+bool MemSafetyAnalysis::analyzeMemoryAccessesInLoop() {
+ LoopBlocksRPO RPOT(L);
+ RPOT.perform(LI);
+ // Bottom-up traversal: process a block after its successors so that
+ // Case#2 propagation has the successor info ready.
+ std::stack<BasicBlock *> WorkList;
+ for (BasicBlock *BB : RPOT)
+ WorkList.push(BB);
+ while (!WorkList.empty()) {
+ if (!processBlock(WorkList.top())) {
+ clearLocalMemory();
+ return false;
+ }
+ WorkList.pop();
+ }
+ IsAnalysisValid = true;
+ clearLocalMemory();
+ return true;
+}
+
+void MemSafetyAnalysis::printAnalysis() const {
+ LLVM_DEBUG(dbgs() << "MemSafetyAnalysis\n");
+ for (auto *Block : L->getBlocks()) {
+ for (auto &Inst : *Block) {
+ Value *Ptr = nullptr;
+ if (auto *Ld = dyn_cast<LoadInst>(&Inst)) {
+ if (!Ld->isSimple())
+ continue;
+ Ptr = Ld->getPointerOperand();
+ } else if (auto *St = dyn_cast<StoreInst>(&Inst)) {
+ if (!St->isSimple())
+ continue;
+ Ptr = St->getPointerOperand();
+ } else {
+ continue;
+ }
+ const SCEV *PtrSCEV = SE->getSCEV(Ptr);
+ if (isGuaranteedMemoryAccess(PtrSCEV)) {
+ LLVM_DEBUG(dbgs() << " [Guaranteed]: " << Inst
+ << " SCEV: " << *PtrSCEV << "\n");
+ }
+ }
+ }
+}
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index cdccd04f4c9ea..084fd5680c1c5 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -720,6 +720,10 @@ bool TargetTransformInfo::enableMaskedInterleavedAccessVectorization() const {
return TTIImpl->enableMaskedInterleavedAccessVectorization();
}
+bool TargetTransformInfo::shouldRewriteMaskedStoreAsLoadBlendStore() const {
+ return TTIImpl->shouldRewriteMaskedStoreAsLoadBlendStore();
+}
+
bool TargetTransformInfo::isFPVectorizationPotentiallyUnsafe() const {
return TTIImpl->isFPVectorizationPotentiallyUnsafe();
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 24934cb01cdd1..66d06594a0677 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -335,6 +335,17 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
return isLegalMaskedLoadStore(DataType, Alignment);
}
+ bool shouldRewriteMaskedStoreAsLoadBlendStore() const override {
+ // NEON has no masked-store instruction, so the loop vectorizer otherwise
+ // scalarizes a predicated store into a per-lane extract/branch/store
+ // ladder. Rewriting a guaranteed-safe masked store as an unconditional
+ // load + select + unconditional store removes that ladder entirely.
+ // Enable the rewrite only when SVE / Streaming-SVE is unavailable: with
+ // SVE the masked store lowers to a single predicated instruction, so the
+ // rewrite is neutral there (see RFC OptimizeMaskedMemory 7.2).
+ return !ST->isSVEorStreamingSVEAvailable();
+ }
+
bool isElementTypeLegalForCompressStore(Type *Ty) const {
return Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isIntegerTy(32) ||
Ty->isIntegerTy(64);
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 9e833344e7341..9623d5c4ce2ea 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -502,6 +502,13 @@ def TuningSlowSHLD : SubtargetFeature<"slow-shld", "IsSHLDSlow", "true",
"SHLD instruction is slow",
[], InlineIgnore>;
+def TuningSlowMaskedStore
+ : SubtargetFeature<"slow-masked-store", "HasSlowMaskedStore", "true",
+ "Variable-mask vector store is microcoded / slow "
+ "(e.g. AMD znver1/2/3 vmaskmovps/vpmaskmovd path); "
+ "prefer rewriting safe masked stores as unconditional "
+ "load+select+store">;
+
def TuningSlowPMULLD : SubtargetFeature<"slow-pmulld", "IsPMULLDSlow", "true",
"PMULLD instruction is slow (compared to PMULLW/PMULHW and PMULUDQ)",
[], InlineIgnore>;
@@ -1679,6 +1686,7 @@ def ProcessorFeatures {
TuningFastImm16,
TuningSlowDivide64,
TuningSlowSHLD,
+ TuningSlowMaskedStore,
TuningSBBDepBreaking,
TuningInsertVZEROUPPER,
TuningAllowLight256Bit];
@@ -1701,8 +1709,11 @@ def ProcessorFeatures {
!listconcat(ZN2Features, ZN3AdditionalFeatures);
list<SubtargetFeature> ZN4AdditionalTuning = [TuningFastDPWSSD];
+ // znver4 (and later) has first-class AVX-512 masked-store hardware; drop the
+ // tuning that says variable-mask stores are slow.
list<SubtargetFeature> ZN4Tuning =
- !listconcat(ZN3Tuning, ZN4AdditionalTuning);
+ !listremove(!listconcat(ZN3Tuning, ZN4AdditionalTuning),
+ [TuningSlowMaskedStore]);
list<SubtargetFeature> ZN4AdditionalFeatures = [FeatureAVX512,
FeatureCDI,
FeatureDQI,
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 8838fd7e71f02..60f5a72357c84 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -6851,6 +6851,14 @@ bool X86TTIImpl::enableInterleavedAccessVectorization() const {
return !(ST->isAtom());
}
+bool X86TTIImpl::shouldRewriteMaskedStoreAsLoadBlendStore() const {
+ // Driven by the X86 tuning feature `slow-masked-store`, which is set on
+ // znver1/znver2/znver3 (the AMD Zen generations whose variable-mask vector
+ // store path is microcoded). znver4 and later have first-class AVX-512
+ // masked stores and therefore opt out via X86.td.
+ return ST->hasSlowMaskedStore();
+}
+
bool X86TTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
switch (II->getIntrinsicID()) {
default:
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 22171f5469d98..e023b7104fa9c 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -252,6 +252,8 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
bool supportsEfficientVectorElementLo...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/206654
More information about the llvm-commits
mailing list