[llvm] [IR][RFC] Add @llvm.mask.beforefirst intrinsic (PR #203874)
Luke Lau via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 00:40:26 PDT 2026
https://github.com/lukel97 updated https://github.com/llvm/llvm-project/pull/203874
>From bc2497b5279529e3602e01afc7bb94b362fcfced Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 15 Jun 2026 18:07:27 +0800
Subject: [PATCH 1/8] [IR][RFC] Add @llvm.mask.beforefirst intrinsic
---
llvm/docs/LangRef.rst | 40 ++++++
llvm/include/llvm/CodeGen/ISDOpcodes.h | 4 +
llvm/include/llvm/IR/Intrinsics.td | 5 +
.../include/llvm/Target/TargetSelectionDAG.td | 5 +
.../SelectionDAG/LegalizeIntegerTypes.cpp | 17 +--
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 3 +-
.../SelectionDAG/LegalizeVectorOps.cpp | 29 ++++
.../SelectionDAG/SelectionDAGBuilder.cpp | 6 +
.../SelectionDAG/SelectionDAGDumper.cpp | 2 +
llvm/lib/CodeGen/TargetLoweringBase.cpp | 2 +
llvm/lib/IR/Verifier.cpp | 5 +
.../Target/AArch64/AArch64ISelLowering.cpp | 1 +
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 9 ++
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +
.../Target/RISCV/RISCVInstrInfoVSDPatterns.td | 4 +
.../CodeGen/AArch64/mask-beforefirst-sve.ll | 121 ++++++++++++++++
llvm/test/CodeGen/AArch64/mask-beforefirst.ll | 90 ++++++++++++
.../rvv/fixed-vectors-mask-beforefirst.ll | 134 ++++++++++++++++++
.../CodeGen/RISCV/rvv/mask-beforefirst.ll | 80 +++++++++++
llvm/test/Verifier/mask-beforefirst.ll | 9 ++
20 files changed, 554 insertions(+), 14 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
create mode 100644 llvm/test/CodeGen/AArch64/mask-beforefirst.ll
create mode 100644 llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-beforefirst.ll
create mode 100644 llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll
create mode 100644 llvm/test/Verifier/mask-beforefirst.ll
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index e570c9a385e8f..519ef6db68c75 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -22004,6 +22004,46 @@ Examples:
; because if any more lanes were active the load would be dependent on the
; completion of the store.
+.. _int_mask_beforefirst
+
+'``llvm.mask.beforefirst.*``' Intrinsic
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+This is an overloaded intrinsic.
+
+::
+
+ declare <4 x i1> @llvm.mask.beforefirst.v4i1(<4 x i1> %mask)
+ declare <vscale x 8 x i1> @llvm.mask.beforefirst.nxv8i1(<vscale x 8 x i1> %mask)
+
+
+Overview:
+"""""""""
+
+Given a vector mask, returns a new mask with all elements before the first active element in the input set to 1, and every element afterwards set to 0.
+
+Arguments:
+""""""""""
+
+Takes one argument which must be an i1 vector, and returns a vector of the same type.
+
+Semantics:
+""""""""""
+
+When the input is all zeroes, the result is all ones.
+
+Examples:
+"""""""""
+
+.. code-block:: llvm
+ @llvm.mask.beforefirst(<0,0,1,1>); ==> <1,1,0,0>
+ @llvm.mask.beforefirst(<0,0,0,0>); ==> <1,1,1,1>
+ @llvm.mask.beforefirst(<0,1,0,1>); ==> <1,0,0,0>
+ @llvm.mask.beforefirst(<1,0,0,1>); ==> <0,0,0,0>
+
+
Experimental Vector Intrinsics
------------------------------
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 11099df775e38..c9ac660d1bcd0 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -1603,6 +1603,10 @@ enum NodeType {
/// bits conform to getBooleanContents similar to the SETCC operator.
GET_ACTIVE_LANE_MASK,
+ /// Has one mask vector operand. Returns a mask with all ones up to but not
+ /// including the first active element in the input, followed by zeroes.
+ MASK_BEFOREFIRST,
+
/// The `llvm.loop.dependence.{war, raw}.mask` intrinsics
/// Operands: Load pointer, Store pointer, Element size, Lane offset
/// Output: Mask
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 7ae8e3d513ed0..323b8894f0517 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2615,6 +2615,11 @@ def int_get_active_lane_mask:
[llvm_anyint_ty, LLVMMatchType<1>],
[IntrNoMem, IntrSpeculatable]>;
+def int_mask_beforefirst:
+ DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+ [LLVMMatchType<0>],
+ [IntrNoMem, IntrSpeculatable]>;
+
def int_experimental_get_vector_length:
DefaultAttrsIntrinsic<[llvm_i32_ty],
[llvm_anyint_ty, llvm_i32_ty, llvm_i1_ty],
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 8ed2698f12b1b..936bc58a137f8 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -919,6 +919,11 @@ def get_active_lane_mask
SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisInt<1>, SDTCisSameAs<1, 2>]>,
[]>;
+def mask_beforefirst
+ : SDNode<"ISD::MASK_BEFOREFIRST",
+ SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisSameAs<0, 1>]>,
+ []>;
+
// Nodes for intrinsics, you should use the intrinsic itself and let tblgen use
// these internally. Don't reference these directly.
def intrinsic_void : SDNode<"ISD::INTRINSIC_VOID",
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index a5faff4f038cd..3472842ed5d58 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -162,11 +162,9 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
case ISD::VECTOR_FIND_LAST_ACTIVE:
- Res = PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(N);
- break;
-
+ case ISD::MASK_BEFOREFIRST:
case ISD::GET_ACTIVE_LANE_MASK:
- Res = PromoteIntRes_GET_ACTIVE_LANE_MASK(N);
+ Res = PromoteIntRes_BooleanVector(N);
break;
case ISD::PARTIAL_REDUCE_UMLA:
@@ -2195,6 +2193,7 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
case ISD::VECTOR_FIND_LAST_ACTIVE:
case ISD::CTTZ_ELTS:
case ISD::CTTZ_ELTS_ZERO_POISON:
+ case ISD::MASK_BEFOREFIRST:
Res = PromoteIntOp_UnaryBooleanVectorOp(N, OpNo);
break;
case ISD::GET_ACTIVE_LANE_MASK:
@@ -6482,16 +6481,10 @@ SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
}
-SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
- EVT VT = N->getValueType(0);
- EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
- return DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, SDLoc(N), NVT, N->ops());
-}
-
-SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
+SDValue DAGTypeLegalizer::PromoteIntRes_BooleanVector(SDNode *N) {
EVT VT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
- return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops());
+ return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, N->ops());
}
SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 450eba435cc0b..5a81c96320c3c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -362,8 +362,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntRes_IS_FPCLASS(SDNode *N);
SDValue PromoteIntRes_PATCHPOINT(SDNode *N);
SDValue PromoteIntRes_READ_REGISTER(SDNode *N);
- SDValue PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
- SDValue PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N);
+ SDValue PromoteIntRes_BooleanVector(SDNode *N);
SDValue PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N);
SDValue PromoteIntRes_LOOP_DEPENDENCE_MASK(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 03e36b3ea75e8..75ffe8f7e5b90 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -29,6 +29,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/SelectionDAG.h"
@@ -139,6 +140,7 @@ class VectorLegalizer {
SDValue ExpandVP_FABS(SDNode *Node);
SDValue ExpandVP_FCOPYSIGN(SDNode *Node);
SDValue ExpandLOOP_DEPENDENCE_MASK(SDNode *N);
+ SDValue ExpandMASK_BEFOREFIRST(SDNode *N);
SDValue ExpandMaskedBinOp(SDNode *N);
SDValue ExpandSELECT(SDNode *Node);
std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
@@ -484,6 +486,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
case ISD::UCMP:
case ISD::LOOP_DEPENDENCE_WAR_MASK:
case ISD::LOOP_DEPENDENCE_RAW_MASK:
+ case ISD::MASK_BEFOREFIRST:
case ISD::MASKED_UDIV:
case ISD::MASKED_SDIV:
case ISD::MASKED_UREM:
@@ -1398,6 +1401,9 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
case ISD::LOOP_DEPENDENCE_RAW_MASK:
Results.push_back(ExpandLOOP_DEPENDENCE_MASK(Node));
return;
+ case ISD::MASK_BEFOREFIRST:
+ Results.push_back(ExpandMASK_BEFOREFIRST(Node));
+ return;
case ISD::FADD:
case ISD::FMUL:
@@ -1917,6 +1923,29 @@ SDValue VectorLegalizer::ExpandLOOP_DEPENDENCE_MASK(SDNode *N) {
return TLI.expandLoopDependenceMask(N, DAG);
}
+SDValue VectorLegalizer::ExpandMASK_BEFOREFIRST(SDNode *N) {
+ // Expand to (setcc ult (step-vector), (cttz_elts x))
+ SDLoc DL(N);
+ EVT VT = N->getValueType(0);
+
+ // Compute the type for the cttz_elts.
+ ConstantRange VScaleRange(1, /*isFullSet=*/true); // Fixed length default.
+ if (VT.isScalableVector())
+ VScaleRange = getVScaleRange(&DAG.getMachineFunction().getFunction(), 64);
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ uint64_t EltWidth = TLI.getBitWidthForCttzElements(
+ EVT(TLI.getVectorIdxTy(DAG.getDataLayout())), VT.getVectorElementCount(),
+ /*ZeroIsPoison=*/false, &VScaleRange);
+
+ EVT EltVT = EVT::getIntegerVT(*DAG.getContext(), EltWidth);
+ EVT CttzEltsVT =
+ EVT::getVectorVT(*DAG.getContext(), EltVT, VT.getVectorElementCount());
+ SDValue CttzElts = DAG.getSplatVector(
+ CttzEltsVT, DL, DAG.getNode(ISD::CTTZ_ELTS, DL, EltVT, N->getOperand(0)));
+ SDValue StepVector = DAG.getStepVector(DL, CttzEltsVT);
+ return DAG.getSetCC(DL, VT, StepVector, CttzElts, ISD::CondCode::SETULT);
+}
+
SDValue VectorLegalizer::ExpandMaskedBinOp(SDNode *N) {
// Masked bin ops don't have undefined behaviour when dividing by zero
// on disabled lanes and produce poison instead. Replace the divisor on the
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 52af36014c5b3..fb12d254e9010 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -8302,6 +8302,12 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
setValue(&I, SetCC);
return;
}
+ case Intrinsic::mask_beforefirst: {
+ SDValue Op = getValue(I.getOperand(0));
+ setValue(&I,
+ DAG.getNode(ISD::MASK_BEFOREFIRST, sdl, Op.getValueType(), Op));
+ return;
+ }
case Intrinsic::experimental_get_vector_length: {
assert(cast<ConstantInt>(I.getOperand(1))->getSExtValue() > 0 &&
"Expected positive VF");
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 7b500fc863b72..8ae7f521e45ad 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -602,6 +602,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::GET_ACTIVE_LANE_MASK:
return "get_active_lane_mask";
+ case ISD::MASK_BEFOREFIRST:
+ return "mask_beforefirst";
case ISD::PARTIAL_REDUCE_UMLA:
return "partial_reduce_umla";
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 83ad156e770f4..0f8592dc17a2a 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1257,6 +1257,8 @@ void TargetLoweringBase::initActions() {
setOperationAction(ISD::LOOP_DEPENDENCE_RAW_MASK, VT, Expand);
setOperationAction(ISD::LOOP_DEPENDENCE_WAR_MASK, VT, Expand);
+ setOperationAction(ISD::MASK_BEFOREFIRST, VT, Expand);
+
// FP environment operations default to expand.
setOperationAction(ISD::GET_FPENV, VT, Expand);
setOperationAction(ISD::SET_FPENV, VT, Expand);
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 6d900ce0d917f..e2a78549d563c 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -6675,6 +6675,11 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
Call);
break;
}
+ case Intrinsic::mask_beforefirst: {
+ Check(Call.getType()->getScalarType()->isIntegerTy(1),
+ "mask.beforefirst element type must be i1", Call);
+ break;
+ }
case Intrinsic::experimental_get_vector_length: {
ConstantInt *VF = cast<ConstantInt>(Call.getArgOperand(1));
Check(!VF->isNegative() && !VF->isZero(),
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index faf102f300659..94f12fa183160 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1594,6 +1594,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
Custom);
setOperationAction(ISD::VECTOR_FIND_LAST_ACTIVE, VT, Legal);
setOperationAction(ISD::GET_ACTIVE_LANE_MASK, VT, Legal);
+ setOperationAction(ISD::MASK_BEFOREFIRST, VT, Legal);
}
if (Subtarget->isSVEorStreamingSVEAvailable() &&
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 4712406e37e6b..4ffdc1bcef1fb 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -1143,6 +1143,15 @@ let Predicates = [HasSVE_or_SME] in {
defm BRKB_PPmP : sve_int_break_m<0b101, "brkb", int_aarch64_sve_brkb>;
defm BRKBS_PPzP : sve_int_break_z<0b110, "brkbs", null_frag>;
+ def : Pat<(nxv16i1 (mask_beforefirst PPR8:$Pn)),
+ (BRKB_PPzP (PTRUE_B 31), PPR8:$Pn)>;
+ def : Pat<(nxv8i1 (mask_beforefirst PPR8:$Pn)),
+ (BRKB_PPzP (PTRUE_H 31), PPR8:$Pn)>;
+ def : Pat<(nxv4i1 (mask_beforefirst PPR8:$Pn)),
+ (BRKB_PPzP (PTRUE_S 31), PPR8:$Pn)>;
+ def : Pat<(nxv2i1 (mask_beforefirst PPR8:$Pn)),
+ (BRKB_PPzP (PTRUE_D 31), PPR8:$Pn)>;
+
defm PTEST_PP : sve_int_ptest<0b010000, "ptest", AArch64ptest, AArch64ptest_any, AArch64ptest_first>;
defm PFALSE : sve_int_pfalse<0b000000, "pfalse">;
defm PFIRST : sve_int_pfirst<0b00000, "pfirst", int_aarch64_sve_pfirst>;
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 4a937fe2c6462..008a14981821f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1043,6 +1043,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
setOperationPromotedToType(
{ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT,
MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));
+
+ setOperationAction(ISD::MASK_BEFOREFIRST, VT, Legal);
}
for (MVT VT : IntVecVTs) {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 3c8ab6bad5452..1ac47725adf20 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -1254,6 +1254,10 @@ foreach mti = AllMasks in {
def : Pat<(mti.Mask rvv_vmnot_op:$rs),
(!cast<Instruction>("PseudoVMNAND_MM_"#mti.BX)
VR:$rs, VR:$rs, mti.AVL, mti.Log2SEW)>;
+
+ def : Pat<(mti.Mask (mask_beforefirst VR:$vs2)),
+ (!cast<Instruction>("PseudoVMSBF_M_"#mti.BX)
+ VR:$vs2, mti.AVL, mti.Log2SEW)>;
}
}
diff --git a/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll b/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
new file mode 100644
index 0000000000000..2fe1d44a459bf
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
@@ -0,0 +1,121 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple aarch64 -mattr=+sve | FileCheck %s
+
+define <vscale x 16 x i1> @nxv16i1(<vscale x 16 x i1> %m) {
+; CHECK-LABEL: nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.b
+; CHECK-NEXT: brkb p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %x = call <vscale x 16 x i1> @llvm.mask.beforefirst(<vscale x 16 x i1> %m)
+ ret <vscale x 16 x i1> %x
+}
+
+define <vscale x 8 x i1> @nxv8i1(<vscale x 8 x i1> %m) {
+; CHECK-LABEL: nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: brkb p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %x = call <vscale x 8 x i1> @llvm.mask.beforefirst(<vscale x 8 x i1> %m)
+ ret <vscale x 8 x i1> %x
+}
+
+define <vscale x 4 x i1> @nxv4i1(<vscale x 4 x i1> %m) {
+; CHECK-LABEL: nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: brkb p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %x = call <vscale x 4 x i1> @llvm.mask.beforefirst(<vscale x 4 x i1> %m)
+ ret <vscale x 4 x i1> %x
+}
+
+define <vscale x 2 x i1> @nxv2i1(<vscale x 2 x i1> %m) {
+; CHECK-LABEL: nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: brkb p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %x = call <vscale x 2 x i1> @llvm.mask.beforefirst(<vscale x 2 x i1> %m)
+ ret <vscale x 2 x i1> %x
+}
+
+define <16 x i1> @v16i1(<16 x i1> %m) {
+; CHECK-LABEL: v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-NEXT: mov w8, #16 // =0x10
+; CHECK-NEXT: index z1.b, w8, #-1
+; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: umaxv b0, v0.16b
+; CHECK-NEXT: fmov w9, s0
+; CHECK-NEXT: index z0.b, #0, #1
+; CHECK-NEXT: sub w8, w8, w9, uxtb
+; CHECK-NEXT: dup v1.16b, w8
+; CHECK-NEXT: cmhi v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: ret
+ %x = call <16 x i1> @llvm.mask.beforefirst(<16 x i1> %m)
+ ret <16 x i1> %x
+}
+
+define <8 x i1> @v8i1(<8 x i1> %m) {
+; CHECK-LABEL: v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8b, v0.8b, #7
+; CHECK-NEXT: index z1.b, #8, #-1
+; CHECK-NEXT: mov w9, #8 // =0x8
+; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: index z1.b, #0, #1
+; CHECK-NEXT: umaxv b0, v0.8b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: sub w8, w9, w8, uxtb
+; CHECK-NEXT: dup v0.8b, w8
+; CHECK-NEXT: cmhi v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+ %x = call <8 x i1> @llvm.mask.beforefirst(<8 x i1> %m)
+ ret <8 x i1> %x
+}
+
+define <4 x i1> @v4i1(<4 x i1> %m) {
+; CHECK-LABEL: v4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4h, v0.4h, #15
+; CHECK-NEXT: index z1.h, #4, #-1
+; CHECK-NEXT: mov w9, #4 // =0x4
+; CHECK-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: index z1.h, #0, #1
+; CHECK-NEXT: umaxv h0, v0.4h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: sub w8, w9, w8
+; CHECK-NEXT: dup v0.4h, w8
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: cmhi v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ret
+ %x = call <4 x i1> @llvm.mask.beforefirst(<4 x i1> %m)
+ ret <4 x i1> %x
+}
+
+define <2 x i1> @v2i1(<2 x i1> %m) {
+; CHECK-LABEL: v2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2s, v0.2s, #31
+; CHECK-NEXT: index z1.s, #2, #-1
+; CHECK-NEXT: mov w9, #2 // =0x2
+; CHECK-NEXT: index z2.s, #0, #1
+; CHECK-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: movi d1, #0x0000ff000000ff
+; CHECK-NEXT: umaxp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: sub w8, w9, w8
+; CHECK-NEXT: dup v0.2s, w8
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: cmhi v0.2s, v0.2s, v2.2s
+; CHECK-NEXT: ret
+ %x = call <2 x i1> @llvm.mask.beforefirst(<2 x i1> %m)
+ ret <2 x i1> %x
+}
diff --git a/llvm/test/CodeGen/AArch64/mask-beforefirst.ll b/llvm/test/CodeGen/AArch64/mask-beforefirst.ll
new file mode 100644
index 0000000000000..698627a8a36df
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/mask-beforefirst.ll
@@ -0,0 +1,90 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple aarch64 | FileCheck %s
+
+define <16 x i1> @v16i1(<16 x i1> %m) {
+; CHECK-LABEL: v16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.16b, v0.16b, #7
+; CHECK-NEXT: adrp x8, .LCPI0_0
+; CHECK-NEXT: mov w9, #16 // =0x10
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_0]
+; CHECK-NEXT: cmlt v0.16b, v0.16b, #0
+; CHECK-NEXT: and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: umaxv b0, v0.16b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: sub w8, w9, w8, uxtb
+; CHECK-NEXT: dup v0.16b, w8
+; CHECK-NEXT: adrp x8, .LCPI0_1
+; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_1]
+; CHECK-NEXT: cmhi v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %x = call <16 x i1> @llvm.mask.beforefirst(<16 x i1> %m)
+ ret <16 x i1> %x
+}
+
+define <8 x i1> @v8i1(<8 x i1> %m) {
+; CHECK-LABEL: v8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.8b, v0.8b, #7
+; CHECK-NEXT: adrp x8, .LCPI1_0
+; CHECK-NEXT: mov w9, #8 // =0x8
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI1_0]
+; CHECK-NEXT: cmlt v0.8b, v0.8b, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: umaxv b0, v0.8b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: sub w8, w9, w8, uxtb
+; CHECK-NEXT: dup v0.8b, w8
+; CHECK-NEXT: adrp x8, .LCPI1_1
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI1_1]
+; CHECK-NEXT: cmhi v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ret
+ %x = call <8 x i1> @llvm.mask.beforefirst(<8 x i1> %m)
+ ret <8 x i1> %x
+}
+
+define <4 x i1> @v4i1(<4 x i1> %m) {
+; CHECK-LABEL: v4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.4h, v0.4h, #15
+; CHECK-NEXT: adrp x8, .LCPI2_0
+; CHECK-NEXT: mov w9, #4 // =0x4
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI2_0]
+; CHECK-NEXT: cmlt v0.4h, v0.4h, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: umaxv h0, v0.4h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: sub w8, w9, w8
+; CHECK-NEXT: dup v0.4h, w8
+; CHECK-NEXT: adrp x8, .LCPI2_1
+; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI2_1]
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: cmhi v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ret
+ %x = call <4 x i1> @llvm.mask.beforefirst(<4 x i1> %m)
+ ret <4 x i1> %x
+}
+
+define <2 x i1> @v2i1(<2 x i1> %m) {
+; CHECK-LABEL: v2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: shl v0.2s, v0.2s, #31
+; CHECK-NEXT: mov x8, #2 // =0x2
+; CHECK-NEXT: mov w9, #2 // =0x2
+; CHECK-NEXT: movk x8, #1, lsl #32
+; CHECK-NEXT: fmov d1, x8
+; CHECK-NEXT: cmlt v0.2s, v0.2s, #0
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: movi d1, #0x0000ff000000ff
+; CHECK-NEXT: umaxp v0.2s, v0.2s, v0.2s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: sub w8, w9, w8
+; CHECK-NEXT: dup v0.2s, w8
+; CHECK-NEXT: mov x8, #4294967296 // =0x100000000
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: fmov d1, x8
+; CHECK-NEXT: cmhi v0.2s, v0.2s, v1.2s
+; CHECK-NEXT: ret
+ %x = call <2 x i1> @llvm.mask.beforefirst(<2 x i1> %m)
+ ret <2 x i1> %x
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-beforefirst.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-beforefirst.ll
new file mode 100644
index 0000000000000..ab4cdfc311285
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-mask-beforefirst.ll
@@ -0,0 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple riscv32 -mattr=+v | FileCheck %s
+; RUN: llc < %s -mtriple riscv64 -mattr=+v | FileCheck %s
+
+define <1 x i1> @v1i1(<1 x i1> %m) {
+; CHECK-LABEL: v1i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 1, e8, mf8, ta, ma
+; CHECK-NEXT: vfirst.m a0, v0
+; CHECK-NEXT: bgez a0, .LBB0_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: li a0, 1
+; CHECK-NEXT: .LBB0_2:
+; CHECK-NEXT: vmv.v.x v8, a0
+; CHECK-NEXT: vmsgtu.vi v0, v8, 0
+; CHECK-NEXT: ret
+ %x = call <1 x i1> @llvm.mask.beforefirst(<1 x i1> %m)
+ ret <1 x i1> %x
+}
+
+define <2 x i1> @v2i1(<2 x i1> %m) {
+; CHECK-LABEL: v2i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-NEXT: vfirst.m a0, v0
+; CHECK-NEXT: bgez a0, .LBB1_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: li a0, 2
+; CHECK-NEXT: .LBB1_2:
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vmsltu.vx v0, v8, a0
+; CHECK-NEXT: ret
+ %x = call <2 x i1> @llvm.mask.beforefirst(<2 x i1> %m)
+ ret <2 x i1> %x
+}
+
+define <4 x i1> @v4i1(<4 x i1> %m) {
+; CHECK-LABEL: v4i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma
+; CHECK-NEXT: vfirst.m a0, v0
+; CHECK-NEXT: bgez a0, .LBB2_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: li a0, 4
+; CHECK-NEXT: .LBB2_2:
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vmsltu.vx v0, v8, a0
+; CHECK-NEXT: ret
+ %x = call <4 x i1> @llvm.mask.beforefirst(<4 x i1> %m)
+ ret <4 x i1> %x
+}
+
+define <8 x i1> @v8i1(<8 x i1> %m) {
+; CHECK-LABEL: v8i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT: vfirst.m a0, v0
+; CHECK-NEXT: bgez a0, .LBB3_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: li a0, 8
+; CHECK-NEXT: .LBB3_2:
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vmsltu.vx v0, v8, a0
+; CHECK-NEXT: ret
+ %x = call <8 x i1> @llvm.mask.beforefirst(<8 x i1> %m)
+ ret <8 x i1> %x
+}
+
+define <16 x i1> @v16i1(<16 x i1> %m) {
+; CHECK-LABEL: v16i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT: vfirst.m a0, v0
+; CHECK-NEXT: bgez a0, .LBB4_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: li a0, 16
+; CHECK-NEXT: .LBB4_2:
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vmsltu.vx v0, v8, a0
+; CHECK-NEXT: ret
+ %x = call <16 x i1> @llvm.mask.beforefirst(<16 x i1> %m)
+ ret <16 x i1> %x
+}
+
+define <32 x i1> @v32i1(<32 x i1> %m) {
+; CHECK-LABEL: v32i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: li a0, 32
+; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma
+; CHECK-NEXT: vfirst.m a0, v0
+; CHECK-NEXT: bgez a0, .LBB5_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: li a0, 32
+; CHECK-NEXT: .LBB5_2:
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vmsltu.vx v0, v8, a0
+; CHECK-NEXT: ret
+ %x = call <32 x i1> @llvm.mask.beforefirst(<32 x i1> %m)
+ ret <32 x i1> %x
+}
+
+define <64 x i1> @v64i1(<64 x i1> %m) {
+; CHECK-LABEL: v64i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: li a0, 64
+; CHECK-NEXT: vsetvli zero, a0, e8, m4, ta, ma
+; CHECK-NEXT: vfirst.m a0, v0
+; CHECK-NEXT: bgez a0, .LBB6_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: li a0, 64
+; CHECK-NEXT: .LBB6_2:
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vmsltu.vx v0, v8, a0
+; CHECK-NEXT: ret
+ %x = call <64 x i1> @llvm.mask.beforefirst(<64 x i1> %m)
+ ret <64 x i1> %x
+}
+
+define <128 x i1> @v128i1(<128 x i1> %m) {
+; CHECK-LABEL: v128i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: li a0, 128
+; CHECK-NEXT: vsetvli zero, a0, e8, m8, ta, ma
+; CHECK-NEXT: vfirst.m a0, v0
+; CHECK-NEXT: bgez a0, .LBB7_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: li a0, 128
+; CHECK-NEXT: .LBB7_2:
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vmsltu.vx v0, v8, a0
+; CHECK-NEXT: ret
+ %x = call <128 x i1> @llvm.mask.beforefirst(<128 x i1> %m)
+ ret <128 x i1> %x
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll b/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll
new file mode 100644
index 0000000000000..195ddbaff0ba0
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple riscv32 -mattr=+v | FileCheck %s
+; RUN: llc < %s -mtriple riscv64 -mattr=+v | FileCheck %s
+
+define <vscale x 1 x i1> @nxv1i1(<vscale x 1 x i1> %m) {
+; CHECK-LABEL: nxv1i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e8, mf8, ta, ma
+; CHECK-NEXT: vmsbf.m v8, v0
+; CHECK-NEXT: vmv1r.v v0, v8
+; CHECK-NEXT: ret
+ %x = call <vscale x 1 x i1> @llvm.mask.beforefirst(<vscale x 1 x i1> %m)
+ ret <vscale x 1 x i1> %x
+}
+
+define <vscale x 2 x i1> @nxv2i1(<vscale x 2 x i1> %m) {
+; CHECK-LABEL: nxv2i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e8, mf4, ta, ma
+; CHECK-NEXT: vmsbf.m v8, v0
+; CHECK-NEXT: vmv1r.v v0, v8
+; CHECK-NEXT: ret
+ %x = call <vscale x 2 x i1> @llvm.mask.beforefirst(<vscale x 2 x i1> %m)
+ ret <vscale x 2 x i1> %x
+}
+
+define <vscale x 4 x i1> @nxv4i1(<vscale x 4 x i1> %m) {
+; CHECK-LABEL: nxv4i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT: vmsbf.m v8, v0
+; CHECK-NEXT: vmv1r.v v0, v8
+; CHECK-NEXT: ret
+ %x = call <vscale x 4 x i1> @llvm.mask.beforefirst(<vscale x 4 x i1> %m)
+ ret <vscale x 4 x i1> %x
+}
+
+define <vscale x 8 x i1> @nxv8i1(<vscale x 8 x i1> %m) {
+; CHECK-LABEL: nxv8i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT: vmsbf.m v8, v0
+; CHECK-NEXT: vmv.v.v v0, v8
+; CHECK-NEXT: ret
+ %x = call <vscale x 8 x i1> @llvm.mask.beforefirst(<vscale x 8 x i1> %m)
+ ret <vscale x 8 x i1> %x
+}
+
+define <vscale x 16 x i1> @nxv16i1(<vscale x 16 x i1> %m) {
+; CHECK-LABEL: nxv16i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e8, m2, ta, ma
+; CHECK-NEXT: vmsbf.m v8, v0
+; CHECK-NEXT: vmv1r.v v0, v8
+; CHECK-NEXT: ret
+ %x = call <vscale x 16 x i1> @llvm.mask.beforefirst(<vscale x 16 x i1> %m)
+ ret <vscale x 16 x i1> %x
+}
+
+define <vscale x 32 x i1> @nxv32i1(<vscale x 32 x i1> %m) {
+; CHECK-LABEL: nxv32i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e8, m4, ta, ma
+; CHECK-NEXT: vmsbf.m v8, v0
+; CHECK-NEXT: vmv1r.v v0, v8
+; CHECK-NEXT: ret
+ %x = call <vscale x 32 x i1> @llvm.mask.beforefirst(<vscale x 32 x i1> %m)
+ ret <vscale x 32 x i1> %x
+}
+
+define <vscale x 64 x i1> @nxv64i1(<vscale x 64 x i1> %m) {
+; CHECK-LABEL: nxv64i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT: vmsbf.m v8, v0
+; CHECK-NEXT: vmv1r.v v0, v8
+; CHECK-NEXT: ret
+ %x = call <vscale x 64 x i1> @llvm.mask.beforefirst(<vscale x 64 x i1> %m)
+ ret <vscale x 64 x i1> %x
+}
diff --git a/llvm/test/Verifier/mask-beforefirst.ll b/llvm/test/Verifier/mask-beforefirst.ll
new file mode 100644
index 0000000000000..4cdd9ed025782
--- /dev/null
+++ b/llvm/test/Verifier/mask-beforefirst.ll
@@ -0,0 +1,9 @@
+; RUN: not opt -S -passes=verify < %s 2>&1 | FileCheck %s
+
+; Reject llvm.mask.beforefirst with non-mask vector arguments.
+
+define <4 x i8> @v4i8(<4 x i8> %m) {
+; CHECK: mask.beforefirst element type must be i1
+ %res = call <4 x i8> @llvm.mask.beforefirst(<4 x i8> %m)
+ ret <4 x i8> %res
+}
>From f3057cc91bd24d196fc604908adfba5c29a012e8 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Mon, 15 Jun 2026 21:34:45 +0800
Subject: [PATCH 2/8] Fix LangRef.rst build
---
llvm/docs/LangRef.rst | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 519ef6db68c75..7477b26ede094 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -22004,7 +22004,7 @@ Examples:
; because if any more lanes were active the load would be dependent on the
; completion of the store.
-.. _int_mask_beforefirst
+.. _int_mask_beforefirst:
'``llvm.mask.beforefirst.*``' Intrinsic
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -22038,6 +22038,7 @@ Examples:
"""""""""
.. code-block:: llvm
+
@llvm.mask.beforefirst(<0,0,1,1>); ==> <1,1,0,0>
@llvm.mask.beforefirst(<0,0,0,0>); ==> <1,1,1,1>
@llvm.mask.beforefirst(<0,1,0,1>); ==> <1,0,0,0>
>From 5bfaaba428b1e234e1badda1b0e07a2f099e65d0 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 16 Jun 2026 16:07:47 +0800
Subject: [PATCH 3/8] Don't legalize op, promote op during result legalization
---
.../SelectionDAG/LegalizeIntegerTypes.cpp | 27 +++++++++++++++----
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 4 ++-
2 files changed, 25 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 3472842ed5d58..d2b36ab39e50e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -162,9 +162,14 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
case ISD::VECTOR_FIND_LAST_ACTIVE:
- case ISD::MASK_BEFOREFIRST:
+ Res = PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(N);
+ break;
+
case ISD::GET_ACTIVE_LANE_MASK:
- Res = PromoteIntRes_BooleanVector(N);
+ Res = PromoteIntRes_GET_ACTIVE_LANE_MASK(N);
+ break;
+ case ISD::MASK_BEFOREFIRST:
+ Res = PromoteIntRes_MASK_BEFOREFIRST(N);
break;
case ISD::PARTIAL_REDUCE_UMLA:
@@ -2193,7 +2198,6 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
case ISD::VECTOR_FIND_LAST_ACTIVE:
case ISD::CTTZ_ELTS:
case ISD::CTTZ_ELTS_ZERO_POISON:
- case ISD::MASK_BEFOREFIRST:
Res = PromoteIntOp_UnaryBooleanVectorOp(N, OpNo);
break;
case ISD::GET_ACTIVE_LANE_MASK:
@@ -6481,10 +6485,23 @@ SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
}
-SDValue DAGTypeLegalizer::PromoteIntRes_BooleanVector(SDNode *N) {
+SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N) {
+ EVT VT = N->getValueType(0);
+ EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+ return DAG.getNode(ISD::VECTOR_FIND_LAST_ACTIVE, SDLoc(N), NVT, N->ops());
+}
+
+SDValue DAGTypeLegalizer::PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N) {
+ EVT VT = N->getValueType(0);
+ EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+ return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops());
+}
+
+SDValue DAGTypeLegalizer::PromoteIntRes_MASK_BEFOREFIRST(SDNode *N) {
EVT VT = N->getValueType(0);
EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
- return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, N->ops());
+ SDValue Op = PromoteTargetBoolean(N->getOperand(0), NVT);
+ return DAG.getNode(ISD::MASK_BEFOREFIRST, SDLoc(N), NVT, Op);
}
SDValue DAGTypeLegalizer::PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 5a81c96320c3c..6916bcaa08f35 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -362,7 +362,9 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntRes_IS_FPCLASS(SDNode *N);
SDValue PromoteIntRes_PATCHPOINT(SDNode *N);
SDValue PromoteIntRes_READ_REGISTER(SDNode *N);
- SDValue PromoteIntRes_BooleanVector(SDNode *N);
+ SDValue PromoteIntRes_VECTOR_FIND_LAST_ACTIVE(SDNode *N);
+ SDValue PromoteIntRes_GET_ACTIVE_LANE_MASK(SDNode *N);
+ SDValue PromoteIntRes_MASK_BEFOREFIRST(SDNode *N);
SDValue PromoteIntRes_PARTIAL_REDUCE_MLA(SDNode *N);
SDValue PromoteIntRes_LOOP_DEPENDENCE_MASK(SDNode *N);
>From f1a75a03ea222cb1f0cf18ad88451b8b48756825 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 16 Jun 2026 17:36:27 +0800
Subject: [PATCH 4/8] Fix tablegen indentation
---
llvm/include/llvm/IR/Intrinsics.td | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 323b8894f0517..9de4b0031116a 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2617,8 +2617,8 @@ def int_get_active_lane_mask:
def int_mask_beforefirst:
DefaultAttrsIntrinsic<[llvm_anyvector_ty],
- [LLVMMatchType<0>],
- [IntrNoMem, IntrSpeculatable]>;
+ [LLVMMatchType<0>],
+ [IntrNoMem, IntrSpeculatable]>;
def int_experimental_get_vector_length:
DefaultAttrsIntrinsic<[llvm_i32_ty],
>From fcced5a33f5a35339b012c1fafe3def82d816f65 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 16 Jun 2026 17:42:12 +0800
Subject: [PATCH 5/8] Add widening legalization
---
.../CodeGen/SelectionDAG/LegalizeVectorTypes.cpp | 1 +
llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll | 12 ++++++++++++
llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll | 13 +++++++++++++
3 files changed, 26 insertions(+)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 5350be412176d..c2038e6d20db9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -5478,6 +5478,7 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
case ISD::ARITH_FENCE:
case ISD::FCANONICALIZE:
case ISD::AssertNoFPClass:
+ case ISD::MASK_BEFOREFIRST:
Res = WidenVecRes_Unary(N);
break;
case ISD::FMA: case ISD::VP_FMA:
diff --git a/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll b/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
index 2fe1d44a459bf..c939e8b6af409 100644
--- a/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
+++ b/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
@@ -119,3 +119,15 @@ define <2 x i1> @v2i1(<2 x i1> %m) {
%x = call <2 x i1> @llvm.mask.beforefirst(<2 x i1> %m)
ret <2 x i1> %x
}
+
+; Widening
+
+define <vscale x 3 x i1> @nxv3i1(<vscale x 3 x i1> %m) {
+; CHECK-LABEL: nxv3i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: brkb p0.b, p1/z, p0.b
+; CHECK-NEXT: ret
+ %x = call <vscale x 3 x i1> @llvm.mask.beforefirst(<vscale x 3 x i1> %m)
+ ret <vscale x 3 x i1> %x
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll b/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll
index 195ddbaff0ba0..e2db80b6a9d1d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll
@@ -78,3 +78,16 @@ define <vscale x 64 x i1> @nxv64i1(<vscale x 64 x i1> %m) {
%x = call <vscale x 64 x i1> @llvm.mask.beforefirst(<vscale x 64 x i1> %m)
ret <vscale x 64 x i1> %x
}
+
+; Widening
+
+define <vscale x 3 x i1> @nxv3i1(<vscale x 3 x i1> %m) {
+; CHECK-LABEL: nxv3i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT: vmsbf.m v8, v0
+; CHECK-NEXT: vmv1r.v v0, v8
+; CHECK-NEXT: ret
+ %x = call <vscale x 3 x i1> @llvm.mask.beforefirst(<vscale x 3 x i1> %m)
+ ret <vscale x 3 x i1> %x
+}
>From 646e953e83f8ce42d0ed914c6897776a9051b697 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Tue, 16 Jun 2026 18:06:21 +0800
Subject: [PATCH 6/8] Add splitting legalization
---
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 19 +++++++++++++++++++
.../CodeGen/AArch64/mask-beforefirst-sve.ll | 18 ++++++++++++++++++
.../CodeGen/RISCV/rvv/mask-beforefirst.ll | 19 +++++++++++++++++++
4 files changed, 57 insertions(+)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 6916bcaa08f35..28f120a80e987 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -937,6 +937,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo, SDValue &Hi);
+ void SplitVecRes_MASK_BEFOREFIRST(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index c2038e6d20db9..024b1858b95b1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1335,6 +1335,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::LOOP_DEPENDENCE_WAR_MASK:
SplitVecRes_LOOP_DEPENDENCE_MASK(N, Lo, Hi);
break;
+ case ISD::MASK_BEFOREFIRST:
+ SplitVecRes_MASK_BEFOREFIRST(N, Lo, Hi);
+ break;
case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
case ISD::AssertZext: SplitVecRes_AssertZext(N, Lo, Hi); break;
case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
@@ -1882,6 +1885,22 @@ void DAGTypeLegalizer::SplitVecRes_LOOP_DEPENDENCE_MASK(SDNode *N, SDValue &Lo,
/*LaneOffset=*/DAG.getConstant(LaneOffset, DL, MVT::i64));
}
+void DAGTypeLegalizer::SplitVecRes_MASK_BEFOREFIRST(SDNode *N, SDValue &Lo,
+ SDValue &Hi) {
+ SDLoc DL(N);
+ SDValue InLo, InHi;
+ GetSplitVector(N->getOperand(0), InLo, InHi);
+ EVT VT = InLo.getValueType();
+ Lo = DAG.getNode(ISD::MASK_BEFOREFIRST, DL, VT, InLo);
+
+ // hi = AnyLoActive ? all-ones : (mask_beforefirst hi)
+ SDValue AnyLoActive = DAG.getNode(ISD::VECREDUCE_OR, DL, MVT::i1, Lo);
+ SDValue Cond = DAG.getBoolExtOrTrunc(AnyLoActive, DL,
+ getSetCCResultType(MVT::i1), MVT::i1);
+ Hi = DAG.getNode(ISD::SELECT, DL, VT, Cond, DAG.getConstant(0, DL, VT),
+ DAG.getNode(ISD::MASK_BEFOREFIRST, DL, VT, InHi));
+}
+
void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
SDValue &Hi) {
EVT LoVT, HiVT;
diff --git a/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll b/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
index c939e8b6af409..eeddfd24197bf 100644
--- a/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
+++ b/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
@@ -131,3 +131,21 @@ define <vscale x 3 x i1> @nxv3i1(<vscale x 3 x i1> %m) {
%x = call <vscale x 3 x i1> @llvm.mask.beforefirst(<vscale x 3 x i1> %m)
ret <vscale x 3 x i1> %x
}
+
+; Splitting
+
+define <vscale x 32 x i1> @nxv32i1(<vscale x 32 x i1> %m) {
+; CHECK-LABEL: nxv32i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p2.b
+; CHECK-NEXT: brkb p0.b, p2/z, p0.b
+; CHECK-NEXT: brkb p1.b, p2/z, p1.b
+; CHECK-NEXT: ptest p0, p0.b
+; CHECK-NEXT: cset w8, ne
+; CHECK-NEXT: sbfx x8, x8, #0, #1
+; CHECK-NEXT: whilelo p3.b, xzr, x8
+; CHECK-NEXT: bic p1.b, p1/z, p1.b, p3.b
+; CHECK-NEXT: ret
+ %x = call <vscale x 32 x i1> @llvm.mask.beforefirst(<vscale x 32 x i1> %m)
+ ret <vscale x 32 x i1> %x
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll b/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll
index e2db80b6a9d1d..aafc6b4690109 100644
--- a/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/mask-beforefirst.ll
@@ -91,3 +91,22 @@ define <vscale x 3 x i1> @nxv3i1(<vscale x 3 x i1> %m) {
%x = call <vscale x 3 x i1> @llvm.mask.beforefirst(<vscale x 3 x i1> %m)
ret <vscale x 3 x i1> %x
}
+
+; Splitting
+
+define <vscale x 128 x i1> @nxv128i1(<vscale x 128 x i1> %m) {
+; CHECK-LABEL: nxv128i1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e8, m8, ta, ma
+; CHECK-NEXT: vmsbf.m v9, v0
+; CHECK-NEXT: vcpop.m a0, v9
+; CHECK-NEXT: snez a0, a0
+; CHECK-NEXT: vmv.v.x v16, a0
+; CHECK-NEXT: vmsne.vi v10, v16, 0
+; CHECK-NEXT: vmsbf.m v11, v8
+; CHECK-NEXT: vmandn.mm v8, v11, v10
+; CHECK-NEXT: vmv1r.v v0, v9
+; CHECK-NEXT: ret
+ %x = call <vscale x 128 x i1> @llvm.mask.beforefirst(<vscale x 128 x i1> %m)
+ ret <vscale x 128 x i1> %x
+}
>From 59fa5b2e108e1966e97286d83ba6534731eb89fd Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Wed, 17 Jun 2026 14:52:27 +0200
Subject: [PATCH 7/8] Fix all-ones typo -> all-zeros
---
llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 024b1858b95b1..aca019a9632de 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1893,7 +1893,7 @@ void DAGTypeLegalizer::SplitVecRes_MASK_BEFOREFIRST(SDNode *N, SDValue &Lo,
EVT VT = InLo.getValueType();
Lo = DAG.getNode(ISD::MASK_BEFOREFIRST, DL, VT, InLo);
- // hi = AnyLoActive ? all-ones : (mask_beforefirst hi)
+ // hi = AnyLoActive ? all-zeros : (mask_beforefirst hi)
SDValue AnyLoActive = DAG.getNode(ISD::VECREDUCE_OR, DL, MVT::i1, Lo);
SDValue Cond = DAG.getBoolExtOrTrunc(AnyLoActive, DL,
getSetCCResultType(MVT::i1), MVT::i1);
>From 4668e853ac6f0777632485f23d002805ff538f4a Mon Sep 17 00:00:00 2001
From: Luke Lau <luke at igalia.com>
Date: Sun, 19 Jul 2026 15:40:02 +0800
Subject: [PATCH 8/8] Update sve test
---
llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll b/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
index eeddfd24197bf..1bd814f4ef84b 100644
--- a/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
+++ b/llvm/test/CodeGen/AArch64/mask-beforefirst-sve.ll
@@ -141,8 +141,7 @@ define <vscale x 32 x i1> @nxv32i1(<vscale x 32 x i1> %m) {
; CHECK-NEXT: brkb p0.b, p2/z, p0.b
; CHECK-NEXT: brkb p1.b, p2/z, p1.b
; CHECK-NEXT: ptest p0, p0.b
-; CHECK-NEXT: cset w8, ne
-; CHECK-NEXT: sbfx x8, x8, #0, #1
+; CHECK-NEXT: csetm x8, ne
; CHECK-NEXT: whilelo p3.b, xzr, x8
; CHECK-NEXT: bic p1.b, p1/z, p1.b, p3.b
; CHECK-NEXT: ret
More information about the llvm-commits
mailing list