[llvm] [llvm][optimization] Missed optimization : masked memory operations (PR #225028)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 01:44:32 PDT 2026
https://github.com/SunilKuravinakop created https://github.com/llvm/llvm-project/pull/225028
Avoid masked memory operations when every element of the mask is false.
This also fixes [#224823](https://github.com/llvm/llvm-project/issues/224823)
>From 0d108e6f7d94b22b638963fb50048b005406da10 Mon Sep 17 00:00:00 2001
From: Sunil Kuravinakop <koops at hpe.com>
Date: Mon, 21 Sep 2026 02:02:59 -0500
Subject: [PATCH] Avoid masked memory operations when every element of the mask
is false.
---
.../Transforms/Vectorize/VectorCombine.cpp | 156 +++++++++++++++++-
.../VectorCombine/guard-masked-regions.ll | 139 ++++++++++++++++
.../VectorCombine/guard-multi-mask-regions.ll | 129 +++++++++++++++
3 files changed, 423 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/VectorCombine/guard-masked-regions.ll
create mode 100644 llvm/test/Transforms/VectorCombine/guard-multi-mask-regions.ll
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index e8c5885f9b1e3..649945f29cba4 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -37,6 +37,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include <numeric>
@@ -63,6 +64,10 @@ static cl::opt<bool> DisableVectorCombine(
"disable-vector-combine", cl::init(false), cl::Hidden,
cl::desc("Disable all vector combine transforms"));
+static cl::opt<bool> GuardMaskedRegions(
+ "vector-combine-guard-masked-regions", cl::init(false), cl::Hidden,
+ cl::desc("Experimentally skip masked regions when all lanes are inactive"));
+
static cl::opt<bool> DisableBinopExtractShuffle(
"disable-binop-extract-shuffle", cl::init(false), cl::Hidden,
cl::desc("Disable binop extract to shuffle transforms"));
@@ -7083,6 +7088,151 @@ bool VectorCombine::run() {
return MadeChange;
}
+static bool isCombinedMask(Value *V, ArrayRef<Value *> Masks) {
+ SmallVector<Value *> Worklist{V};
+ SmallPtrSet<Value *, 8> Seen;
+ SmallPtrSet<Value *, 8> Found;
+ while (!Worklist.empty()) {
+ Value *Part = Worklist.pop_back_val();
+ if (!Seen.insert(Part).second)
+ continue;
+ if (is_contained(Masks, Part)) {
+ Found.insert(Part);
+ continue;
+ }
+ Value *LHS, *RHS;
+ if (!match(Part, m_Or(m_Value(LHS), m_Value(RHS))))
+ return false;
+ Worklist.push_back(LHS);
+ Worklist.push_back(RHS);
+ }
+ return Found.size() == Masks.size();
+}
+
+// Keep this opt-in until the cost of the extra branch can be modeled.
+static bool guardMaskedRegions(Function &F) {
+ if (F.hasOptSize())
+ return false;
+ SmallVector<BasicBlock *> Blocks;
+ for (BasicBlock &BB : F)
+ Blocks.push_back(&BB);
+ bool Changed = false;
+ for (BasicBlock *BB : Blocks) {
+ Instruction *First = nullptr, *Last = nullptr;
+ SmallVector<Value *, 4> Masks;
+ SmallVector<IntrinsicInst *> MemoryOps;
+ bool Reject = false;
+ for (Instruction &I : *BB) {
+ auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II || (II->getIntrinsicID() != Intrinsic::masked_load &&
+ II->getIntrinsicID() != Intrinsic::masked_store))
+ continue;
+ bool Store = II->getIntrinsicID() == Intrinsic::masked_store;
+ if (!First)
+ First = II;
+ MemoryOps.push_back(II);
+ if (Store)
+ Last = II;
+ }
+ if (!Last)
+ continue;
+ while (MemoryOps.back() != Last)
+ MemoryOps.pop_back();
+ if (MemoryOps.size() < 2)
+ continue;
+ for (IntrinsicInst *II : MemoryOps) {
+ Value *Mask = II->getArgOperand(
+ II->getIntrinsicID() == Intrinsic::masked_store ? 2 : 1);
+ if (isa<Constant>(Mask) || !isa<FixedVectorType>(Mask->getType()) ||
+ (!Masks.empty() && Mask->getType() != Masks.front()->getType())) {
+ Reject = true;
+ break;
+ }
+ // Do not hoist masks computed inside the region into its guard.
+ if (auto *MI = dyn_cast<Instruction>(Mask))
+ if (MI->getParent() == BB && !MI->comesBefore(First)) {
+ Reject = true;
+ break;
+ }
+ if (!is_contained(Masks, Mask))
+ Masks.push_back(Mask);
+ }
+ if (Reject)
+ continue;
+
+ // Recognize both the original guard and its bitcast/compare canonical form.
+ if (BasicBlock *Pred = BB->getSinglePredecessor()) {
+ auto *BI = dyn_cast<CondBrInst>(Pred->getTerminator());
+ if (BI) {
+ Value *Cond = BI->getCondition();
+ Value *Combined;
+ if (BI->getSuccessor(0) == BB &&
+ (match(Cond, m_Intrinsic<Intrinsic::vector_reduce_or>(
+ m_Value(Combined))) ||
+ match(Cond,
+ m_SpecificICmp(ICmpInst::ICMP_NE,
+ m_BitCast(m_Value(Combined)), m_Zero()))) &&
+ isCombinedMask(Combined, Masks))
+ continue;
+ if (BI->getSuccessor(1) == BB &&
+ match(Cond,
+ m_SpecificICmp(ICmpInst::ICMP_EQ,
+ m_BitCast(m_Value(Combined)), m_Zero())) &&
+ isCombinedMask(Combined, Masks))
+ continue;
+ }
+ }
+
+ SmallVector<Instruction *> Region;
+ SmallPtrSet<Instruction *, 32> Members;
+ for (Instruction *I = First; I; I = I->getNextNode()) {
+ Region.push_back(I);
+ Members.insert(I);
+ if (I == Last)
+ break;
+ }
+ for (Instruction *I : Region) {
+ auto *II = dyn_cast<IntrinsicInst>(I);
+ bool Masked = II && (II->getIntrinsicID() == Intrinsic::masked_load ||
+ II->getIntrinsicID() == Intrinsic::masked_store);
+ if (!Masked && (I->mayReadOrWriteMemory() || I->mayHaveSideEffects() ||
+ I->isTerminator()))
+ Reject = true;
+ if (auto *CB = dyn_cast<CallBase>(I))
+ if (CB->isConvergent())
+ Reject = true;
+ for (User *U : I->users())
+ if (!Members.contains(dyn_cast<Instruction>(U)))
+ Reject = true;
+ }
+ if (Reject)
+ continue;
+
+ IRBuilder<> Builder(First);
+ // The branch and memory operations must agree even for undef mask lanes.
+ SmallDenseMap<Value *, Value *, 4> FrozenMasks;
+ Value *Combined = nullptr;
+ for (Value *Mask : Masks) {
+ Value *Frozen = Builder.CreateFreeze(Mask, "active.mask");
+ FrozenMasks[Mask] = Frozen;
+ Combined = Combined ? Builder.CreateOr(Combined, Frozen, "combined.mask")
+ : Frozen;
+ }
+ Value *Any = Builder.CreateOrReduce(Combined);
+ for (IntrinsicInst *II : MemoryOps) {
+ unsigned MaskIdx =
+ II->getIntrinsicID() == Intrinsic::masked_store ? 2 : 1;
+ II->setArgOperand(MaskIdx,
+ FrozenMasks.lookup(II->getArgOperand(MaskIdx)));
+ }
+ Instruction *ThenTerm = SplitBlockAndInsertIfThen(Any, First, false);
+ for (Instruction *I : Region)
+ I->moveBeforePreserving(ThenTerm->getIterator());
+ Changed = true;
+ }
+ return Changed;
+}
+
PreservedAnalyses VectorCombinePass::run(Function &F,
FunctionAnalysisManager &FAM) {
auto &AC = FAM.getResult<AssumptionAnalysis>(F);
@@ -7093,7 +7243,11 @@ PreservedAnalyses VectorCombinePass::run(Function &F,
TTI::TargetCostKind CostKind =
F.hasOptSize() ? TTI::TCK_CodeSize : TTI::TCK_RecipThroughput;
VectorCombine Combiner(F, TTI, DT, AA, AC, DL, CostKind, TryEarlyFoldsOnly);
- if (!Combiner.run())
+ bool Changed = Combiner.run();
+ if (!DisableVectorCombine && !TryEarlyFoldsOnly && GuardMaskedRegions &&
+ guardMaskedRegions(F))
+ return PreservedAnalyses::none();
+ if (!Changed)
return PreservedAnalyses::all();
PreservedAnalyses PA;
PA.preserveSet<CFGAnalyses>();
diff --git a/llvm/test/Transforms/VectorCombine/guard-masked-regions.ll b/llvm/test/Transforms/VectorCombine/guard-masked-regions.ll
new file mode 100644
index 0000000000000..ce6fe69ab5068
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/guard-masked-regions.ll
@@ -0,0 +1,139 @@
+; RUN: opt -passes=vector-combine -vector-combine-guard-masked-regions -verify-each -S %s | FileCheck %s
+; RUN: opt -passes='vector-combine,vector-combine' -vector-combine-guard-masked-regions -verify-each -S %s | FileCheck %s
+; RUN: opt -passes=vector-combine -S %s | FileCheck %s --check-prefix=OFF
+; RUN: opt -passes=vector-combine -disable-vector-combine -vector-combine-guard-masked-regions -S %s | FileCheck %s --check-prefix=OFF
+; RUN: opt -passes='vector-combine,instcombine,simplifycfg,vector-combine' -vector-combine-guard-masked-regions -verify-each -S %s | FileCheck %s --check-prefix=CANON
+; RUN: opt -passes='debugify,function(vector-combine),check-debugify' -vector-combine-guard-masked-regions -disable-output %s 2>&1 | FileCheck %s --check-prefix=DEBUG
+
+; DEBUG-NOT: WARNING
+; DEBUG: CheckModuleDebugify: PASS
+; CANON-LABEL: define void @guard(
+; CANON: br i1
+; CANON-NOT: br i1
+; CANON: ret void
+
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <8 x float> @llvm.masked.load.v8f32.p0(ptr, i32 immarg, <8 x i1>, <8 x float>)
+declare void @llvm.masked.store.v8f32.p0(<8 x float>, ptr, i32 immarg, <8 x i1>)
+declare void @side_effect()
+declare <8 x float> @collective(<8 x float>) convergent nounwind willreturn memory(none)
+declare token @llvm.experimental.convergence.entry()
+
+; Skip the load, arithmetic and store when all lanes are inactive. Freeze the
+; mask so the guard and memory operations agree even for undefined lanes.
+; OFF-NOT: br i1
+; CHECK-LABEL: define void @guard(
+; CHECK: [[MASK:%.*]] = freeze <8 x i1> %mask
+; CHECK-NEXT: [[ANY:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[MASK]])
+; CHECK-NEXT: br i1 [[ANY]], label %[[BODY:.*]], label %[[EXIT:.*]]
+; CHECK: [[BODY]]:
+; CHECK-NOT: br i1
+; CHECK: [[LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %src, <8 x i1> [[MASK]], <8 x float> poison)
+; CHECK: [[SUM:%.*]] = fadd <8 x float> [[LOAD]], [[LOAD]]
+; CHECK: call void @llvm.masked.store.v8f32.p0(<8 x float> [[SUM]], ptr align 4 %dst, <8 x i1> [[MASK]])
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+define void @guard(ptr %src, ptr %dst, <8 x i1> %mask) #0 {
+ %v = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %src, i32 4, <8 x i1> %mask, <8 x float> poison)
+ %sum = fadd <8 x float> %v, %v
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %sum, ptr %dst, i32 4, <8 x i1> %mask)
+ ret void
+}
+
+; Reject a region whose loaded value escapes through the return: skipping the
+; region would require providing a value for that external use.
+; CHECK-LABEL: define <8 x float> @escaping_value(
+; CHECK-NOT: br i1
+; CHECK: ret <8 x float>
+define <8 x float> @escaping_value(ptr %src, ptr %dst, <8 x i1> %mask) #0 {
+ %v = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %src, i32 4, <8 x i1> %mask, <8 x float> poison)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr %dst, i32 4, <8 x i1> %mask)
+ ret <8 x float> %v
+}
+
+; Skip only when both masks are inactive. Combine frozen masks for the guard,
+; but retain each operation's own mask rather than enabling extra accesses.
+; CHECK-LABEL: define void @different_masks(
+; CHECK: [[A:%.*]] = freeze <8 x i1> %a
+; CHECK: [[B:%.*]] = freeze <8 x i1> %b
+; CHECK: [[AB:%.*]] = or <8 x i1> [[A]], [[B]]
+; CHECK: [[ANYAB:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[AB]])
+; CHECK: br i1 [[ANYAB]], label %[[BODYAB:.*]], label %[[EXITAB:.*]]
+; CHECK: [[BODYAB]]:
+; CHECK-NOT: br i1
+; CHECK: [[V:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %src, <8 x i1> [[A]], <8 x float> poison)
+; CHECK: call void @llvm.masked.store.v8f32.p0(<8 x float> [[V]], ptr align 4 %dst, <8 x i1> [[B]])
+; CHECK: ret void
+define void @different_masks(ptr %src, ptr %dst, <8 x i1> %a, <8 x i1> %b) #0 {
+ %v = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %src, i32 4, <8 x i1> %a, <8 x float> poison)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr %dst, i32 4, <8 x i1> %b)
+ ret void
+}
+
+; Reject an intervening call that may have observable effects even when all
+; mask lanes are inactive.
+; CHECK-LABEL: define void @side_effects(
+; CHECK-NOT: br i1
+; CHECK: ret void
+define void @side_effects(ptr %src, ptr %dst, <8 x i1> %mask) #0 {
+ %v = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %src, i32 4, <8 x i1> %mask, <8 x float> poison)
+ call void @side_effect()
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr %dst, i32 4, <8 x i1> %mask)
+ ret void
+}
+
+; Reject the region because the unmasked store must execute independently of
+; the vector mask.
+; CHECK-LABEL: define void @unmasked_store(
+; CHECK-NOT: br i1
+; CHECK: ret void
+define void @unmasked_store(ptr %src, ptr %dst, ptr %other, <8 x i1> %mask) #0 {
+ %v = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %src, i32 4, <8 x i1> %mask, <8 x float> poison)
+ store i32 1, ptr %other
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr %dst, i32 4, <8 x i1> %mask)
+ ret void
+}
+
+; Avoid adding guard control flow when the function is optimized for code size.
+; CHECK-LABEL: define void @size_optimized(
+; CHECK-NOT: br i1
+; CHECK: ret void
+define void @size_optimized(ptr %src, ptr %dst, <8 x i1> %mask) #1 {
+ %v = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %src, i32 4, <8 x i1> %mask, <8 x float> poison)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr %dst, i32 4, <8 x i1> %mask)
+ ret void
+}
+
+; Guarding a memory-free convergent call can change collective participation.
+; CHECK-LABEL: define void @convergent_call(
+; CHECK-NOT: br i1
+; CHECK: ret void
+; CANON-LABEL: define void @convergent_call(
+; CANON-NOT: br i1
+; CANON: ret void
+define void @convergent_call(ptr %src, ptr %dst, <8 x i1> %mask) convergent #0 {
+ %v = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %src, i32 4, <8 x i1> %mask, <8 x float> zeroinitializer)
+ %sum = call <8 x float> @collective(<8 x float> %v)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %sum, ptr %dst, i32 4, <8 x i1> %mask)
+ ret void
+}
+
+; An explicit convergence token does not make the extra guard safe.
+; CHECK-LABEL: define void @controlled_convergent_call(
+; CHECK-NOT: br i1
+; CHECK: ret void
+; CANON-LABEL: define void @controlled_convergent_call(
+; CANON-NOT: br i1
+; CANON: ret void
+define void @controlled_convergent_call(ptr %src, ptr %dst, <8 x i1> %mask) convergent #0 {
+ %token = call token @llvm.experimental.convergence.entry()
+ %v = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %src, i32 4, <8 x i1> %mask, <8 x float> zeroinitializer)
+ %sum = call <8 x float> @collective(<8 x float> %v) [ "convergencectrl"(token %token) ]
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %sum, ptr %dst, i32 4, <8 x i1> %mask)
+ ret void
+}
+
+attributes #0 = { "target-features"="+avx2" }
+attributes #1 = { optsize "target-features"="+avx2" }
\ No newline at end of file
diff --git a/llvm/test/Transforms/VectorCombine/guard-multi-mask-regions.ll b/llvm/test/Transforms/VectorCombine/guard-multi-mask-regions.ll
new file mode 100644
index 0000000000000..58a3e025cba6d
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/guard-multi-mask-regions.ll
@@ -0,0 +1,129 @@
+; RUN: opt -passes=vector-combine -vector-combine-guard-masked-regions -verify-each -S %s | FileCheck %s
+; RUN: opt -passes='vector-combine,vector-combine' -vector-combine-guard-masked-regions -verify-each -S %s | FileCheck %s
+; RUN: opt -passes='vector-combine,instcombine,simplifycfg,vector-combine' -vector-combine-guard-masked-regions -verify-each -S %s | FileCheck %s --check-prefix=CANON
+; RUN: opt -passes=vector-combine -S %s | FileCheck %s --check-prefix=OFF
+; RUN: opt -passes='debugify,function(vector-combine),check-debugify' -vector-combine-guard-masked-regions -disable-output %s 2>&1 | FileCheck %s --check-prefix=DEBUG
+
+; DEBUG-NOT: WARNING
+; DEBUG: CheckModuleDebugify: PASS
+; OFF-NOT: br i1
+; CANON-LABEL: define void @interleaved(
+; CANON: br i1
+; CANON-NOT: br i1
+; CANON: ret void
+
+target triple = "x86_64-unknown-linux-gnu"
+
+declare <8 x float> @llvm.masked.load.v8f32.p0(ptr, <8 x i1>, <8 x float>)
+declare void @llvm.masked.store.v8f32.p0(<8 x float>, ptr, <8 x i1>)
+declare <4 x float> @llvm.masked.load.v4f32.p0(ptr, <4 x i1>, <4 x float>)
+declare void @llvm.masked.store.v4f32.p0(<4 x float>, ptr, <4 x i1>)
+declare void @side_effect()
+
+; Guard both interleaved load/store pairs together, skipping them only when
+; both masks are inactive. Each access retains its own frozen mask.
+; CHECK-LABEL: define void @interleaved(
+; CHECK: [[A:%.*]] = freeze <8 x i1> %a
+; CHECK: [[B:%.*]] = freeze <8 x i1> %b
+; CHECK: [[AB:%.*]] = or <8 x i1> [[A]], [[B]]
+; CHECK: [[ANY:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[AB]])
+; CHECK: br i1 [[ANY]], label %[[BODY:.*]], label %[[EXIT:.*]]
+; CHECK: [[BODY]]:
+; CHECK-NOT: br i1
+; CHECK: [[VA:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %srca, <8 x i1> [[A]], <8 x float> zeroinitializer)
+; CHECK: [[VB:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %srcb, <8 x i1> [[B]], <8 x float> zeroinitializer)
+; CHECK: call void @llvm.masked.store.v8f32.p0(<8 x float> [[VA]], ptr align 4 %dsta, <8 x i1> [[A]])
+; CHECK: call void @llvm.masked.store.v8f32.p0(<8 x float> [[VB]], ptr align 4 %dstb, <8 x i1> [[B]])
+; CHECK: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK: ret void
+define void @interleaved(ptr %srca, ptr %srcb, ptr %dsta, ptr %dstb, <8 x i1> %a, <8 x i1> %b) #0 {
+ %va = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %srca, <8 x i1> %a, <8 x float> zeroinitializer)
+ %vb = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %srcb, <8 x i1> %b, <8 x float> zeroinitializer)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %va, ptr align 4 %dsta, <8 x i1> %a)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %vb, ptr align 4 %dstb, <8 x i1> %b)
+ ret void
+}
+
+; Reject a mask computed from a load inside the region: it is not available
+; before the region to form the combined guard.
+; CHECK-LABEL: define void @mask_inside_region(
+; CHECK-NOT: br i1
+; CHECK: ret void
+define void @mask_inside_region(ptr %src, ptr %dst, <8 x i1> %a) #0 {
+ %v = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %src, <8 x i1> %a, <8 x float> zeroinitializer)
+ %b = fcmp ogt <8 x float> %v, zeroinitializer
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr align 4 %dst, <8 x i1> %b)
+ ret void
+}
+
+; Reject a loaded value used by an unmasked store after the region. Skipping
+; the region would leave that external use without its value.
+; CHECK-LABEL: define void @escaping_multimask(
+; CHECK-NOT: br i1
+; CHECK: ret void
+define void @escaping_multimask(ptr %srca, ptr %srcb, ptr %dst, ptr %other, <8 x i1> %a, <8 x i1> %b) #0 {
+ %va = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %srca, <8 x i1> %a, <8 x float> zeroinitializer)
+ %vb = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %srcb, <8 x i1> %b, <8 x float> zeroinitializer)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %vb, ptr align 4 %dst, <8 x i1> %b)
+ store <8 x float> %va, ptr %other
+ ret void
+}
+
+; Reject an intervening call whose effects must occur even when both masks
+; are inactive.
+; CHECK-LABEL: define void @multimask_side_effect(
+; CHECK-NOT: br i1
+; CHECK: ret void
+define void @multimask_side_effect(ptr %src, ptr %dst, <8 x i1> %a, <8 x i1> %b) #0 {
+ %v = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %src, <8 x i1> %a, <8 x float> zeroinitializer)
+ call void @side_effect()
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr align 4 %dst, <8 x i1> %b)
+ ret void
+}
+
+; Reject masks with different lane counts: the combined-mask construction
+; requires matching fixed-vector types.
+; CHECK-LABEL: define void @different_widths(
+; CHECK-NOT: br i1
+; CHECK: ret void
+define void @different_widths(ptr %srca, ptr %srcb, ptr %dsta, ptr %dstb, <8 x i1> %a, <4 x i1> %b) #0 {
+ %va = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 4 %srca, <8 x i1> %a, <8 x float> zeroinitializer)
+ %vb = call <4 x float> @llvm.masked.load.v4f32.p0(ptr align 4 %srcb, <4 x i1> %b, <4 x float> zeroinitializer)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %va, ptr align 4 %dsta, <8 x i1> %a)
+ call void @llvm.masked.store.v4f32.p0(<4 x float> %vb, ptr align 4 %dstb, <4 x i1> %b)
+ ret void
+}
+
+; Combine all four store masks into one guard while preserving individual
+; masks. Recognize the guard after simplification to avoid adding another.
+; CHECK-LABEL: define void @four_masks(
+; CHECK: [[A4:%.*]] = freeze <8 x i1> %a
+; CHECK: [[B4:%.*]] = freeze <8 x i1> %b
+; CHECK: [[AB4:%.*]] = or <8 x i1> [[A4]], [[B4]]
+; CHECK: [[C4:%.*]] = freeze <8 x i1> %c
+; CHECK: [[ABC4:%.*]] = or <8 x i1> [[AB4]], [[C4]]
+; CHECK: [[D4:%.*]] = freeze <8 x i1> %d
+; CHECK: [[ABCD4:%.*]] = or <8 x i1> [[ABC4]], [[D4]]
+; CHECK: [[ANY4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[ABCD4]])
+; CHECK: br i1 [[ANY4]], label %[[BODY4:.*]], label %[[EXIT4:.*]]
+; CHECK: [[BODY4]]:
+; CHECK-NOT: br i1
+; CHECK: call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr align 4 %p, <8 x i1> [[A4]])
+; CHECK: call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr align 4 %q, <8 x i1> [[B4]])
+; CHECK: call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr align 4 %r, <8 x i1> [[C4]])
+; CHECK: call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr align 4 %s, <8 x i1> [[D4]])
+; CHECK: ret void
+; CANON-LABEL: define void @four_masks(
+; CANON: br i1
+; CANON-NOT: br i1
+; CANON: ret void
+define void @four_masks(ptr %p, ptr %q, ptr %r, ptr %s, <8 x float> %v, <8 x i1> %a, <8 x i1> %b, <8 x i1> %c, <8 x i1> %d) #0 {
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr align 4 %p, <8 x i1> %a)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr align 4 %q, <8 x i1> %b)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr align 4 %r, <8 x i1> %c)
+ call void @llvm.masked.store.v8f32.p0(<8 x float> %v, ptr align 4 %s, <8 x i1> %d)
+ ret void
+}
+
+attributes #0 = { "target-features"="+avx2" }
\ No newline at end of file
More information about the llvm-commits
mailing list