[llvm] [VPlan] Constant-fold ActiveLaneMask intrinsics (PR #208852)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 00:43:44 PDT 2026
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/208852
>From 46123b177214e936f54e9800523dce5803b45241 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Sun, 2 Aug 2026 11:53:43 +0100
Subject: [PATCH 1/3] [LV] Pre-commit ALM tests
---
.../vplan-scalable-active-lane-mask.ll | 29 +++++++++++++++++++
.../VPlan/vplan-foldable-active-lane-mask.ll | 26 +++++++++++++++++
2 files changed, 55 insertions(+)
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-scalable-active-lane-mask.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/VPlan/vplan-foldable-active-lane-mask.ll
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-scalable-active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-scalable-active-lane-mask.ll
new file mode 100644
index 0000000000000..10a952683119b
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/AArch64/vplan-scalable-active-lane-mask.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "active.lane.mask" --version 6
+; RUN: opt -mtriple=aarch64 -mcpu=neoverse-v1 -passes=loop-vectorize -force-vector-width="vscale x 4" -force-vector-interleave=1 -tail-folding-policy=prefer-fold-tail -disable-output -vplan-print-after=printOptimizedVPlan %s 2>&1 | FileCheck %s
+
+; We never emit a scalable active.lane.mask that can be folded to a constant.
+define i32 @scalable_alm_liveins(ptr %A, ptr noalias %B) {
+; CHECK-LABEL: VPlan for loop in 'scalable_alm_liveins'
+; CHECK: EMIT vp<%active.lane.mask.entry> = active lane mask vp<%index.part.next>, ir<1002>, ir<1>
+; CHECK: ACTIVE-LANE-MASK-PHI vp<[[VP5:%[0-9]+]]> = phi vp<%active.lane.mask.entry>, vp<%active.lane.mask.next>
+; CHECK: EMIT vp<%active.lane.mask.next> = active lane mask vp<[[VP8:%[0-9]+]]>, ir<1002>, ir<1>
+; CHECK: EMIT vp<[[VP9:%[0-9]+]]> = not vp<%active.lane.mask.next>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %for = phi i16 [ 0, %entry ], [ %for.next, %loop ]
+ %sext = sext i16 %for to i32
+ %iv.next = add i64 %iv, 1
+ %gep.A = getelementptr i32, ptr %A, i64 %iv.next
+ %for.next = load i16, ptr %gep.A, align 2
+ %gep.B = getelementptr i32, ptr %B, i64 %iv.next
+ store i32 %sext, ptr %gep.B
+ %ec = icmp eq i64 %iv, 1001
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i32 0
+}
diff --git a/llvm/test/Transforms/LoopVectorize/VPlan/vplan-foldable-active-lane-mask.ll b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-foldable-active-lane-mask.ll
new file mode 100644
index 0000000000000..b2fdc9e79cad7
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/VPlan/vplan-foldable-active-lane-mask.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --filter "active.lane.mask" --version 6
+; RUN: opt -passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-tail-folding-style=data -force-vector-width=4 -disable-output -vplan-print-after=printOptimizedVPlan %s 2>&1 | FileCheck %s --check-prefix=FIXED
+; RUN: opt -passes=loop-vectorize -tail-folding-policy=must-fold-tail -force-tail-folding-style=data -force-vector-width="vscale x 4" -disable-output -vplan-print-after=printOptimizedVPlan %s 2>&1 | FileCheck %s --check-prefix=SCALABLE
+
+; We never emit fixed-vector active.lane.masks foldable to false.
+define i16 @foldable_fixed_alm(i32 %x) {
+; FIXED-LABEL: VPlan for loop in 'foldable_fixed_alm'
+; SCALABLE-LABEL: VPlan for loop in 'foldable_fixed_alm'
+; SCALABLE: EMIT vp<%active.lane.mask> = active lane mask vp<[[VP5:%[0-9]+]]>, ir<4>, ir<1>
+; SCALABLE: EMIT vp<[[VP7:%[0-9]+]]> = last-active-lane vp<%active.lane.mask>
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %red = phi i32 [ 0, %entry ], [ %add, %loop ]
+ %t = trunc i32 %red to i16
+ %add = add i32 %red, %x
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv, 3
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ ret i16 %t
+}
>From fa9de517cf21adb2efc26f09bc3515e5609c77e4 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 22 Jun 2026 18:18:42 +0100
Subject: [PATCH 2/3] [VPlan] Constant-fold ActiveLaneMask intrinsics
Constant-fold fixed-vector types via llvm::simplifyIntrinsic, making it
accessible from InstSimplifyFolder, and use it for folding
get.active.lane.mask in VPlan.
---
llvm/include/llvm/Analysis/ConstantFolding.h | 1 +
llvm/include/llvm/Analysis/TargetFolder.h | 3 ++-
llvm/lib/Analysis/ConstantFolding.cpp | 6 ++++--
llvm/lib/Analysis/InstructionSimplify.cpp | 2 +-
.../InstCombine/InstCombineSelect.cpp | 2 +-
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 17 +++++++++++++++++
.../AArch64/first-order-recurrence-fold-tail.ll | 3 +--
7 files changed, 27 insertions(+), 7 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index 4ea0a6f69e81b..87867d6f5303e 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -171,6 +171,7 @@ LLVM_ABI Constant *ConstantFoldCall(const CallBase *Call, Function *F,
LLVM_ABI Constant *ConstantFoldIntrinsic(Intrinsic::ID ID,
ArrayRef<Constant *> Ops, Type *Ty,
+ const DataLayout &DL,
Function *CxtF = nullptr);
/// ConstantFoldLoadThroughBitcast - try to cast constant to destination type
diff --git a/llvm/include/llvm/Analysis/TargetFolder.h b/llvm/include/llvm/Analysis/TargetFolder.h
index 9bae2e4d5c974..80d798ee64356 100644
--- a/llvm/include/llvm/Analysis/TargetFolder.h
+++ b/llvm/include/llvm/Analysis/TargetFolder.h
@@ -196,7 +196,8 @@ class LLVM_ABI TargetFolder final : public IRBuilderFolder {
Function *CxtF = nullptr) const override {
if (all_of(Ops, IsaPred<Constant>))
return ConstantFoldIntrinsic(
- ID, ArrayRef((Constant *const *)Ops.data(), Ops.size()), Ty, CxtF);
+ ID, ArrayRef((Constant *const *)Ops.data(), Ops.size()), Ty, DL,
+ CxtF);
return nullptr;
}
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index e5d52ece939c2..748bb3eec9ada 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -4300,7 +4300,7 @@ static Constant *ConstantFoldScalarCall(StringRef Name,
static Constant *ConstantFoldFixedVectorCall(
StringRef Name, Intrinsic::ID IntrinsicID, FixedVectorType *FVTy,
ArrayRef<Constant *> Operands, const DataLayout &DL,
- const TargetLibraryInfo *TLI, const CallBase *Call) {
+ const TargetLibraryInfo *TLI = nullptr, const CallBase *Call = nullptr) {
SmallVector<Constant *, 4> Result(FVTy->getNumElements());
SmallVector<Constant *, 4> Lane(Operands.size());
Type *Ty = FVTy->getElementType();
@@ -4713,13 +4713,15 @@ ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
Constant *llvm::ConstantFoldIntrinsic(Intrinsic::ID ID,
ArrayRef<Constant *> Ops, Type *Ty,
- Function *CxtF) {
+ const DataLayout &DL, Function *CxtF) {
// In the absence of CxtF, assume strictfp conservatively.
if (!canConstantFoldIntrinsic(ID, CxtF ? CxtF->isStrictFP() : true) ||
(DisableFPCallFolding &&
anyTypeContainsFP(
Ty, ArrayRef<Value *>((Value *const *)Ops.data(), Ops.size()))))
return nullptr;
+ if (auto *FVTy = dyn_cast<FixedVectorType>(Ty))
+ return ConstantFoldFixedVectorCall("", ID, FVTy, Ops, DL);
return ConstantFoldScalarCall("", ID, Ty, Ops);
}
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index ec044be705ab0..10b159192c4be 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -7352,7 +7352,7 @@ Value *llvm::simplifyIntrinsic(Intrinsic::ID IID, Type *ReturnType,
if (all_of(Args, IsaPred<Constant>))
if (Constant *C = ConstantFoldIntrinsic(
IID, ArrayRef((Constant *const *)Args.data(), Args.size()),
- ReturnType, CxtF))
+ ReturnType, Q.DL, CxtF))
return C;
// Most of the intrinsics with no operands have some kind of side effect.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 460024f607858..f109d3fd61640 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -2301,7 +2301,7 @@ Value *InstCombinerImpl::foldSelectWithConstOpToBinOp(ICmpInst *Cmp,
auto FoldBinaryOpOrIntrinsic = [&](Constant *LHS, Constant *RHS) {
return IsIntrinsic
- ? ConstantFoldIntrinsic(Opcode, {LHS, RHS}, LHS->getType())
+ ? ConstantFoldIntrinsic(Opcode, {LHS, RHS}, LHS->getType(), DL)
: ConstantFoldBinaryOpOperands(Opcode, LHS, RHS, DL);
};
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 93b18b31e9e7d..b658e57a0e597 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1115,6 +1115,23 @@ VPIRValue *vputils::tryToFoldLiveIns(VPSingleDefRecipe &R,
case Instruction::ExtractElement:
assert(!Ops[0]->getType()->isVectorTy() && "Live-ins should be scalar");
return Ops[0];
+ case VPInstruction::ActiveLaneMask: {
+ uint64_t Multiplier = cast<ConstantInt>(Ops[2])->getZExtValue();
+ Type *I1Ty = IntegerType::getInt1Ty(Plan.getContext());
+ if (Plan.hasScalableVF())
+ // We do not produce foldable scalable ALMs at the moment.
+ return nullptr;
+
+ // We do not produce ALMs foldable to false at the moment.
+ unsigned MaxVF = Multiplier * max_element(Plan.vectorFactors(),
+ ElementCount::isKnownLT)
+ ->getFixedValue();
+ if (auto *C = dyn_cast_if_present<Constant>(Folder.FoldIntrinsic(
+ Intrinsic::get_active_lane_mask, drop_end(Ops),
+ FixedVectorType::get(I1Ty, MaxVF))))
+ if (C->isOneValue())
+ return ConstantInt::getTrue(I1Ty);
+ }
}
return nullptr;
};
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll b/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
index 380554dd22b64..ed4fcb31f9dda 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/first-order-recurrence-fold-tail.ll
@@ -9,11 +9,10 @@ define i32 @test_phi_iterator_invalidation(ptr %A, ptr noalias %B) {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 0, i64 1002)
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ]
-; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE6]] ]
+; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <4 x i1> [ splat (i1 true), [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[PRED_LOAD_CONTINUE6]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_LOAD_CONTINUE6]] ]
; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i16> [ <i16 poison, i16 poison, i16 poison, i16 0>, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[PRED_LOAD_CONTINUE6]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i64> [[VEC_IND]], splat (i64 1)
>From 0dcf2673dc3bd17fdf3255b3d364a310eedab0c8 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <artagnon at tenstorrent.com>
Date: Mon, 3 Aug 2026 08:42:11 +0100
Subject: [PATCH 3/3] [ConstFold] Fix for underlying bug
---
llvm/lib/Analysis/ConstantFolding.cpp | 7 +++++--
.../Transforms/InstSimplify/ConstProp/WebAssembly/dot.ll | 9 +++++++++
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 748bb3eec9ada..c7347aebbd2e3 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -4466,9 +4466,12 @@ static Constant *ConstantFoldFixedVectorCall(
for (unsigned I = 0; I < NumElements; ++I) {
ConstantInt *Elt0 =
- cast<ConstantInt>(Operands[0]->getAggregateElement(I));
+ dyn_cast<ConstantInt>(Operands[0]->getAggregateElement(I));
ConstantInt *Elt1 =
- cast<ConstantInt>(Operands[1]->getAggregateElement(I));
+ dyn_cast<ConstantInt>(Operands[1]->getAggregateElement(I));
+
+ if (!Elt0 || !Elt1)
+ return nullptr;
MulVector[I] = Elt0->getSExtValue() * Elt1->getSExtValue();
}
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/WebAssembly/dot.ll b/llvm/test/Transforms/InstSimplify/ConstProp/WebAssembly/dot.ll
index d44437bd518af..dcb4b32152e8e 100644
--- a/llvm/test/Transforms/InstSimplify/ConstProp/WebAssembly/dot.ll
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/WebAssembly/dot.ll
@@ -28,6 +28,15 @@ define <4 x i32> @dot_nonzero() {
ret <4 x i32> %res
}
+define <4 x i32> @dot_poison() {
+; CHECK-LABEL: define <4 x i32> @dot_poison() {
+; CHECK-NEXT: [[RES:%.*]] = tail call <4 x i32> @llvm.wasm.dot(<8 x i16> <i16 1, i16 poison, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>, <8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>)
+; CHECK-NEXT: ret <4 x i32> [[RES]]
+;
+ %res = tail call <4 x i32> @llvm.wasm.dot(<8 x i16> <i16 1, i16 poison, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>, <8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>)
+ ret <4 x i32> %res
+}
+
define <4 x i32> @dot_one_negative() {
; CHECK-LABEL: define <4 x i32> @dot_one_negative() {
; CHECK-NEXT: ret <4 x i32> splat (i32 -2)
More information about the llvm-commits
mailing list