[llvm] [CodeGen] Split wide active lane mask into smaller masks if preferred (PR #202909)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 02:55:39 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Kerry McLaughlin (kmclaughlin-arm)
<details>
<summary>Changes</summary>
Adds optimizeGetActiveLaneMask to CodeGenPrepare, which will rewrite a
get.active.lane.mask used by a series of subvector extracts if this is the
form preferred by the target.
This PR is part of a series of changes leading to the use of wide active
lane masks as the canonical form in LoopVectorize when choosing to interleave
and tail-fold loops. https://github.com/llvm/llvm-project/pull/193757 is the original PR where this was discussed.
For all targets other than AArch64, the preference is to use multiple smaller
get.active.lane.mask intrinsics, which matches the output of LoopVectorize today
when interleaving unless wide active lane masks are forced. For AArch64,
prefer a wide mask + extracts when either SVE2p1 or SME2 is available.
The motivation for this PR is to leave the decision on which form of the IR
to use up to the target. This will remove the need for more complex decisions
and transformations during vectorisation to query whether it's best to create
a single mask + extracts or multiple smaller masks.
---
Patch is 51.58 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/202909.diff
7 Files Affected:
- (modified) llvm/include/llvm/CodeGen/TargetLowering.h (+2)
- (modified) llvm/lib/CodeGen/CodeGenPrepare.cpp (+100)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+5)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.h (+2)
- (modified) llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll (+119-51)
- (added) llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll (+146)
- (added) llvm/test/CodeGen/Thumb2/get-active-lane-mask-extract.ll (+776)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 82c47cce0f522..288d3c31521b1 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -487,6 +487,8 @@ class LLVM_ABI TargetLoweringBase {
return true;
}
+ virtual bool preferWideActiveLaneMask() const { return false; }
+
virtual bool shouldExpandGetVectorLength(EVT CountVT, unsigned VF,
bool IsScalable) const {
return true;
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 047ca6cebb1c6..82a7da2d51bd9 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -426,6 +426,7 @@ class CodeGenPrepare {
bool optimizeGatherScatterInst(Instruction *MemoryInst, Value *Ptr);
bool optimizeMulWithOverflow(Instruction *I, bool IsSigned,
ModifyDT &ModifiedDT);
+ bool optimizeGetActiveLaneMask(Instruction *I);
bool optimizeInlineAsmInst(CallInst *CS);
bool optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT);
bool optimizeExt(Instruction *&I);
@@ -2834,6 +2835,8 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, ModifyDT &ModifiedDT) {
return optimizeMulWithOverflow(II, /*IsSigned=*/false, ModifiedDT);
case Intrinsic::smul_with_overflow:
return optimizeMulWithOverflow(II, /*IsSigned=*/true, ModifiedDT);
+ case Intrinsic::get_active_lane_mask:
+ return optimizeGetActiveLaneMask(II);
}
SmallVector<Value *, 2> PtrOps;
@@ -6608,6 +6611,103 @@ bool CodeGenPrepare::optimizeMulWithOverflow(Instruction *I, bool IsSigned,
return true;
}
+// Rewrite a get.active.lane.mask intrinsic followed by vector extracts
+// into multiple smaller get.active.lane.mask intrinsics. This should
+// only be rewritten if the target prefers this form over using a
+// wide active lane mask.
+//
+// Example:
+// %alm = call <16 x i1> @llvm.get.active.lane.mask(i32 %i, i32 %n)
+// %p0 = call <8 x i1> @llvm.vector.extract(<16 x i1> %alm, i64 0)
+// %p1 = call <8 x i1> @llvm.vector.extract(<16 x i1> %alm, i64 8)
+//
+// is rewritten as:
+// %alm0 = call <8 x i1> @llvm.get.active.lane.mask(i32 %i, i32 %n)
+// %next = call i32 @llvm.uadd.sat(i32 %i, i32 8)
+// %alm1 = call <8 x i1> @llvm.get.active.lane.mask(i32 %next, i32 %n)
+//
+bool CodeGenPrepare::optimizeGetActiveLaneMask(Instruction *ALM) {
+ if (TLI->preferWideActiveLaneMask())
+ return false;
+
+ // Count the number of users of ALM which are vector extracts
+ unsigned NumExts = count_if(ALM->users(), [](User *U) {
+ auto *II = dyn_cast<IntrinsicInst>(U);
+ return II && II->getIntrinsicID() == Intrinsic::vector_extract;
+ });
+
+ // Return false if the extracts are not the right type for the number
+ // of parts (e.g. 4 x v2i1 extracts of a single v8i1 mask).
+ auto RetTy = cast<VectorType>(ALM->getType());
+ auto MaskEC = RetTy->getElementCount();
+ if (NumExts < 2 || !MaskEC.isKnownMultipleOf(NumExts))
+ return false;
+
+ ElementCount ExtractEC = MaskEC.divideCoefficientBy(NumExts);
+ if (ExtractEC.getKnownMinValue() < 2)
+ return false;
+
+ SmallVector<IntrinsicInst *> Extracts(NumExts, nullptr);
+ for (User *U : ALM->users()) {
+ // The only users accepted are vector extracts and an extract of
+ // element 0.
+ if (auto ExtElt = dyn_cast<ExtractElementInst>(U)) {
+ auto CIdx = dyn_cast<ConstantInt>(ExtElt->getIndexOperand());
+ if (!CIdx || CIdx->getZExtValue() != 0)
+ return false;
+ continue;
+ }
+
+ auto *II = dyn_cast<IntrinsicInst>(U);
+ if (!II || II->getIntrinsicID() != Intrinsic::vector_extract)
+ return false;
+
+ unsigned Idx = cast<ConstantInt>(II->getOperand(1))->getZExtValue();
+ if (cast<VectorType>(II->getType())->getElementCount() != ExtractEC)
+ return false;
+
+ unsigned Part = Idx / ExtractEC.getKnownMinValue();
+ if (Extracts[Part] != nullptr)
+ return false;
+
+ Extracts[Part] = II;
+ }
+
+ IRBuilder<> Builder(ALM);
+ Value *Part0Mask;
+ auto *StartVal = ALM->getOperand(0);
+ auto StartEC = Builder.CreateElementCount(StartVal->getType(), ExtractEC);
+ // Replace each vector extract with a get.active.lane.mask of the same type.
+ for (unsigned I = 0; I < NumExts; ++I) {
+ auto ExtTy = Extracts[0]->getType();
+ if (I > 0)
+ StartVal =
+ Builder.CreateBinaryIntrinsic(Intrinsic::uadd_sat, StartVal, StartEC);
+
+ Value *ALMForPart = Builder.CreateIntrinsic(
+ ExtTy, Intrinsic::get_active_lane_mask, {StartVal, ALM->getOperand(1)});
+ if (I == 0)
+ Part0Mask = ALMForPart;
+ Extracts[I]->replaceAllUsesWith(ALMForPart);
+ Extracts[I]->replaceUsesOfWith(ALM, PoisonValue::get(RetTy));
+ }
+
+ // Update the remaining users, which must be extracts of element 0 from
+ // the original wide mask.
+ SmallVector<User *> RemainingUsers(ALM->users());
+ for (User *U : RemainingUsers) {
+ auto *ExtElt = dyn_cast<ExtractElementInst>(U);
+ assert(ExtElt && "Unexpected user of get.active.lane.mask");
+ auto *NewExt =
+ Builder.CreateExtractElement(Part0Mask, ExtElt->getIndexOperand());
+ ExtElt->replaceAllUsesWith(NewExt);
+ ExtElt->replaceUsesOfWith(ALM, PoisonValue::get(RetTy));
+ }
+ ALM->eraseFromParent();
+
+ return true;
+}
+
/// If there are any memory operands, use OptimizeMemoryInst to sink their
/// address computing into the block when possible / profitable.
bool CodeGenPrepare::optimizeInlineAsmInst(CallInst *CS) {
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 08f3d4e0d30ac..c75ad4ae3102d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2364,6 +2364,11 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
}
}
+bool AArch64TargetLowering::preferWideActiveLaneMask() const {
+ return Subtarget->isSVEorStreamingSVEAvailable() &&
+ (Subtarget->hasSVE2p1() || Subtarget->hasSME2());
+}
+
bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
EVT OpVT) const {
// Only SVE has a 1:1 mapping from intrinsic -> instruction (whilelo).
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 3a93d9a3c0d1e..1f5f0079397ce 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -551,6 +551,8 @@ class AArch64TargetLowering : public TargetLowering {
EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty,
bool AllowUnknown = false) const override;
+ bool preferWideActiveLaneMask() const override;
+
bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override;
bool shouldExpandCttzElements(EVT VT) const override;
diff --git a/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll b/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
index 62cf5d686fd97..82c1b08c67a13 100644
--- a/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
+++ b/llvm/test/CodeGen/AArch64/get-active-lane-mask-extract.ll
@@ -10,9 +10,11 @@ target triple = "aarch64-linux"
define void @test_2x8bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0 {
; CHECK-SVE-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: whilelo p1.b, w0, w1
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: cnth x8
+; CHECK-SVE-NEXT: whilelo p0.h, w0, w1
+; CHECK-SVE-NEXT: adds w8, w0, w8
+; CHECK-SVE-NEXT: csinv w8, w8, wzr, lo
+; CHECK-SVE-NEXT: whilelo p1.h, w8, w1
; CHECK-SVE-NEXT: b use
;
; CHECK-SVE2p1-SME2-LABEL: test_2x8bit_mask_with_32bit_index_and_trip_count:
@@ -31,9 +33,11 @@ define void @test_2x8bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0
define void @test_2x8bit_mask_with_64bit_index_and_trip_count(i64 %i, i64 %n) #0 {
; CHECK-SVE-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: whilelo p1.b, x0, x1
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: cnth x8
+; CHECK-SVE-NEXT: whilelo p0.h, x0, x1
+; CHECK-SVE-NEXT: adds x8, x0, x8
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: whilelo p1.h, x8, x1
; CHECK-SVE-NEXT: b use
;
; CHECK-SVE2p1-SME2-LABEL: test_2x8bit_mask_with_64bit_index_and_trip_count:
@@ -71,9 +75,11 @@ define void @test_edge_case_2x1bit_mask(i64 %i, i64 %n) #0 {
define void @test_boring_case_2x2bit_mask(i64 %i, i64 %n) #0 {
; CHECK-SVE-LABEL: test_boring_case_2x2bit_mask:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: whilelo p1.s, x0, x1
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: cntd x8
+; CHECK-SVE-NEXT: whilelo p0.d, x0, x1
+; CHECK-SVE-NEXT: adds x8, x0, x8
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: whilelo p1.d, x8, x1
; CHECK-SVE-NEXT: b use
;
; CHECK-SVE2p1-SME2-LABEL: test_boring_case_2x2bit_mask:
@@ -90,13 +96,17 @@ define void @test_boring_case_2x2bit_mask(i64 %i, i64 %n) #0 {
define void @test_legal_4x2bit_mask(i64 %i, i64 %n) #0 {
; CHECK-SVE-LABEL: test_legal_4x2bit_mask:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: whilelo p0.h, x0, x1
-; CHECK-SVE-NEXT: punpkhi p1.h, p0.b
-; CHECK-SVE-NEXT: punpklo p4.h, p0.b
-; CHECK-SVE-NEXT: punpkhi p3.h, p1.b
-; CHECK-SVE-NEXT: punpklo p2.h, p1.b
-; CHECK-SVE-NEXT: punpklo p0.h, p4.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p4.b
+; CHECK-SVE-NEXT: cntd x8
+; CHECK-SVE-NEXT: whilelo p0.d, x0, x1
+; CHECK-SVE-NEXT: adds x9, x0, x8
+; CHECK-SVE-NEXT: csinv x9, x9, xzr, lo
+; CHECK-SVE-NEXT: whilelo p1.d, x9, x1
+; CHECK-SVE-NEXT: adds x9, x9, x8
+; CHECK-SVE-NEXT: csinv x9, x9, xzr, lo
+; CHECK-SVE-NEXT: whilelo p2.d, x9, x1
+; CHECK-SVE-NEXT: adds x8, x9, x8
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: whilelo p3.d, x8, x1
; CHECK-SVE-NEXT: b use
;
; CHECK-SVE2p1-SME2-LABEL: test_legal_4x2bit_mask:
@@ -250,9 +260,9 @@ define void @test_2x16bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #
; CHECK-SVE-LABEL: test_2x16bit_mask_with_32bit_index_and_trip_count:
; CHECK-SVE: // %bb.0:
; CHECK-SVE-NEXT: rdvl x8, #1
+; CHECK-SVE-NEXT: whilelo p0.b, w0, w1
; CHECK-SVE-NEXT: adds w8, w0, w8
; CHECK-SVE-NEXT: csinv w8, w8, wzr, lo
-; CHECK-SVE-NEXT: whilelo p0.b, w0, w1
; CHECK-SVE-NEXT: whilelo p1.b, w8, w1
; CHECK-SVE-NEXT: b use
;
@@ -272,18 +282,17 @@ define void @test_2x16bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #
define void @test_2x32bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #0 {
; CHECK-SVE-LABEL: test_2x32bit_mask_with_32bit_index_and_trip_count:
; CHECK-SVE: // %bb.0:
-; CHECK-SVE-NEXT: rdvl x8, #2
-; CHECK-SVE-NEXT: rdvl x9, #1
-; CHECK-SVE-NEXT: adds w8, w0, w8
-; CHECK-SVE-NEXT: csinv w8, w8, wzr, lo
-; CHECK-SVE-NEXT: adds w10, w8, w9
-; CHECK-SVE-NEXT: csinv w10, w10, wzr, lo
-; CHECK-SVE-NEXT: whilelo p3.b, w10, w1
-; CHECK-SVE-NEXT: adds w9, w0, w9
-; CHECK-SVE-NEXT: csinv w9, w9, wzr, lo
+; CHECK-SVE-NEXT: rdvl x8, #1
; CHECK-SVE-NEXT: whilelo p0.b, w0, w1
+; CHECK-SVE-NEXT: adds w9, w0, w8
+; CHECK-SVE-NEXT: csinv w9, w9, wzr, lo
; CHECK-SVE-NEXT: whilelo p1.b, w9, w1
-; CHECK-SVE-NEXT: whilelo p2.b, w8, w1
+; CHECK-SVE-NEXT: adds w9, w9, w8
+; CHECK-SVE-NEXT: csinv w9, w9, wzr, lo
+; CHECK-SVE-NEXT: whilelo p2.b, w9, w1
+; CHECK-SVE-NEXT: adds w8, w9, w8
+; CHECK-SVE-NEXT: csinv w8, w8, wzr, lo
+; CHECK-SVE-NEXT: whilelo p3.b, w8, w1
; CHECK-SVE-NEXT: b use
;
; CHECK-SVE2p1-SME2-LABEL: test_2x32bit_mask_with_32bit_index_and_trip_count:
@@ -310,11 +319,14 @@ define void @test_2x32bit_mask_with_32bit_index_and_trip_count(i32 %i, i32 %n) #
define void @test_2x8bit_mask_with_extracts_and_ptest(i64 %i, i64 %n) {
; CHECK-SVE-LABEL: test_2x8bit_mask_with_extracts_and_ptest:
; CHECK-SVE: // %bb.0: // %entry
-; CHECK-SVE-NEXT: whilelo p1.b, x0, x1
-; CHECK-SVE-NEXT: b.pl .LBB11_2
+; CHECK-SVE-NEXT: cnth x8
+; CHECK-SVE-NEXT: whilelo p0.h, x0, x1
+; CHECK-SVE-NEXT: cset w9, mi
+; CHECK-SVE-NEXT: adds x8, x0, x8
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: cbz w9, .LBB11_2
; CHECK-SVE-NEXT: // %bb.1: // %if.then
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: whilelo p1.h, x8, x1
; CHECK-SVE-NEXT: b use
; CHECK-SVE-NEXT: .LBB11_2: // %if.end
; CHECK-SVE-NEXT: ret
@@ -348,11 +360,14 @@ if.end:
define void @test_2x8bit_mask_with_extracts_and_reinterpret_casts(i64 %i, i64 %n) {
; CHECK-SVE-LABEL: test_2x8bit_mask_with_extracts_and_reinterpret_casts:
; CHECK-SVE: // %bb.0: // %entry
-; CHECK-SVE-NEXT: whilelo p1.h, x0, x1
-; CHECK-SVE-NEXT: b.pl .LBB12_2
+; CHECK-SVE-NEXT: cntw x8
+; CHECK-SVE-NEXT: whilelo p0.s, x0, x1
+; CHECK-SVE-NEXT: cset w9, mi
+; CHECK-SVE-NEXT: adds x8, x0, x8
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: cbz w9, .LBB12_2
; CHECK-SVE-NEXT: // %bb.1: // %if.then
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: whilelo p1.s, x8, x1
; CHECK-SVE-NEXT: b use
; CHECK-SVE-NEXT: .LBB12_2: // %if.end
; CHECK-SVE-NEXT: ret
@@ -383,15 +398,20 @@ if.end:
define void @test_4x4bit_mask_with_extracts_and_ptest(i64 %i, i64 %n) {
; CHECK-SVE-LABEL: test_4x4bit_mask_with_extracts_and_ptest:
; CHECK-SVE: // %bb.0: // %entry
-; CHECK-SVE-NEXT: whilelo p0.b, x0, x1
-; CHECK-SVE-NEXT: b.pl .LBB13_2
+; CHECK-SVE-NEXT: cntw x10
+; CHECK-SVE-NEXT: whilelo p0.s, x0, x1
+; CHECK-SVE-NEXT: cset w11, mi
+; CHECK-SVE-NEXT: adds x8, x0, x10
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: adds x9, x8, x10
+; CHECK-SVE-NEXT: csinv x9, x9, xzr, lo
+; CHECK-SVE-NEXT: adds x10, x9, x10
+; CHECK-SVE-NEXT: csinv x10, x10, xzr, lo
+; CHECK-SVE-NEXT: cbz w11, .LBB13_2
; CHECK-SVE-NEXT: // %bb.1: // %if.then
-; CHECK-SVE-NEXT: punpklo p1.h, p0.b
-; CHECK-SVE-NEXT: punpkhi p3.h, p0.b
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
-; CHECK-SVE-NEXT: punpklo p2.h, p3.b
-; CHECK-SVE-NEXT: punpkhi p3.h, p3.b
+; CHECK-SVE-NEXT: whilelo p1.s, x8, x1
+; CHECK-SVE-NEXT: whilelo p2.s, x9, x1
+; CHECK-SVE-NEXT: whilelo p3.s, x10, x1
; CHECK-SVE-NEXT: b use
; CHECK-SVE-NEXT: .LBB13_2: // %if.end
; CHECK-SVE-NEXT: ret
@@ -428,15 +448,20 @@ if.end:
define void @test_4x2bit_mask_with_extracts_and_reinterpret_casts(i64 %i, i64 %n) {
; CHECK-SVE-LABEL: test_4x2bit_mask_with_extracts_and_reinterpret_casts:
; CHECK-SVE: // %bb.0: // %entry
-; CHECK-SVE-NEXT: whilelo p0.h, x0, x1
-; CHECK-SVE-NEXT: b.pl .LBB14_2
+; CHECK-SVE-NEXT: cntd x10
+; CHECK-SVE-NEXT: whilelo p0.d, x0, x1
+; CHECK-SVE-NEXT: cset w11, mi
+; CHECK-SVE-NEXT: adds x8, x0, x10
+; CHECK-SVE-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SVE-NEXT: adds x9, x8, x10
+; CHECK-SVE-NEXT: csinv x9, x9, xzr, lo
+; CHECK-SVE-NEXT: adds x10, x9, x10
+; CHECK-SVE-NEXT: csinv x10, x10, xzr, lo
+; CHECK-SVE-NEXT: cbz w11, .LBB14_2
; CHECK-SVE-NEXT: // %bb.1: // %if.then
-; CHECK-SVE-NEXT: punpklo p1.h, p0.b
-; CHECK-SVE-NEXT: punpkhi p3.h, p0.b
-; CHECK-SVE-NEXT: punpklo p0.h, p1.b
-; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
-; CHECK-SVE-NEXT: punpklo p2.h, p3.b
-; CHECK-SVE-NEXT: punpkhi p3.h, p3.b
+; CHECK-SVE-NEXT: whilelo p1.d, x8, x1
+; CHECK-SVE-NEXT: whilelo p2.d, x9, x1
+; CHECK-SVE-NEXT: whilelo p3.d, x10, x1
; CHECK-SVE-NEXT: b use
; CHECK-SVE-NEXT: .LBB14_2: // %if.end
; CHECK-SVE-NEXT: ret
@@ -470,6 +495,49 @@ if.end:
ret void
}
+; Test use of get.active.lane.mask by an extractelement with an index value other than 0.
+define void @test_2x8bit_mask_with_invalid_extract_elt(i64 %i, i64 %n) {
+; CHECK-SVE-LABEL: test_2x8bit_mask_with_invalid_extract_elt:
+; CHECK-SVE: // %bb.0: // %entry
+; CHECK-SVE-NEXT: whilelo p1.h, x0, x1
+; CHECK-SVE-NEXT: mov z0.h, p1/z, #1 // =0x1
+; CHECK-SVE-NEXT: mov z0.h, z0.h[8]
+; CHECK-SVE-NEXT: fmov w8, s0
+; CHECK-SVE-NEXT: tbz w8, #0, .LBB15_2
+; CHECK-SVE-NEXT: // %bb.1: // %if.then
+; CHECK-SVE-NEXT: punpklo p0.h, p1.b
+; CHECK-SVE-NEXT: punpkhi p1.h, p1.b
+; CHECK-SVE-NEXT: b use
+; CHECK-SVE-NEXT: .LBB15_2: // %if.end
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SVE2p1-SME2-LABEL: test_2x8bit_mask_with_invalid_extract_elt:
+; CHECK-SVE2p1-SME2: // %bb.0: // %entry
+; CHECK-SVE2p1-SME2-NEXT: whilelo { p0.s, p1.s }, x0, x1
+; CHECK-SVE2p1-SME2-NEXT: uzp1 p2.h, p0.h, p1.h
+; CHECK-SVE2p1-SME2-NEXT: mov z0.h, p2/z, #1 // =0x1
+; CHECK-SVE2p1-SME2-NEXT: mov z0.h, z0.h[8]
+; CHECK-SVE2p1-SME2-NEXT: fmov w8, s0
+; CHECK-SVE2p1-SME2-NEXT: tbz w8, #0, .LBB15_2
+; CHECK-SVE2p1-SME2-NEXT: // %bb.1: // %if.then
+; CHECK-SVE2p1-SME2-NEXT: b use
+; CHECK-SVE2p1-SME2-NEXT: .LBB15_2: // %if.end
+; CHECK-SVE2p1-SME2-NEXT: ret
+entry:
+ %r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %i, i64 %n)
+ %v0 = tail call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> %r, i64 0)
+ %v1 = tail call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1(<vscale x 8 x i1> %r, i64 4)
+ %elt0 = extractelement <vscale x 8 x i1> %r, i64 8
+ br i1 %elt0, label %if.then, label %if.end
+
+if.then:
+ tail call void @use(<vscale x 4 x i1> %v0, <vscale x 4 x i1> %v1)
+ br label %if.end
+
+if.end:
+ ret void
+}
+
declare void @use(...)
attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll b/llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll
new file mode 100644
index 0000000000000..24b9e6d13e44b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/get-active-lane-mask-extract.ll
@@ -0,0 +1,146 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv64 -mattr=+m,+v -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK
+
+define void @test_2x4bit_mask_with_extracts(i64 %i, i64 %n) #0 {
+; CHECK-LABEL: test_2x4bit_mask_with_extracts:
+; CHECK: # %bb.0:
+; CHECK-NEXT: csrr a2, vlenb
+; CHECK-NEXT: vsetvli a3, zero, e64, m4, ta, ma
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: srli a2, a2, 1
+; CHECK-NEXT: vsaddu.vx v12, v8, a0
+; CHECK-NEXT: vmsltu.vx v0, v12, a1
+; CHECK-NEXT: add a2, a0, a2
+; CHECK-NEXT: sltu a0, a2, a0
+; CHECK-NEXT: neg a0, a0
+; CHECK-NEXT: or a0, a0, a2
+; CHECK-NEXT: vsaddu.vx v12, v8, a0
+; CHECK-NEXT: vmsltu.vx v8, v12, a1
+; CHECK-NEXT: tail use
+ %r = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 %i, i64 %n)
+ %v0 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 0)
+ %v1 = call <vscale x 4 x i1> @llvm.vector.extract.nxv4i1.nxv8i1.i64(<vscale x 8 x i1> %r, i64 4)
+ tail call void @use(<vscale x 4 x i1> %v0, <vscale x 4 x i1> %v1)
+ ret void
+}
+
+define void @test_2x8bit_mask_with_extracts(i64 %i, i64 %n) #0 {
+; CHECK-LABEL: test_2x8bit_mask_with_extracts:
+; CHECK: # %bb.0:
+; CHECK-NEXT: csrr a2, vlenb
+; CHECK-NEXT: vsetvli a3, zero, e64, m8, ta, ma
+; CHECK-NEXT: vid.v v8
+; CHECK-NEXT: vsaddu.vx v16, v8, a0
+; CHECK-NEXT: add a2, a0, a2
+; CHECK-NEXT: vmsltu.vx v0, v16, a1
+; CHECK-NEXT: sltu a0, a2, a0
+; CHECK-NEXT: neg a0, a0
+; CHECK-NEXT: or a0, a0, a2
+; CHECK-NEXT: vsaddu.vx v16, v8, a0
+; CHECK-NEXT: vmsltu.vx v8, v16, a1
+; CHECK-NEXT: tail use
+ %r = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 %i, i64 %n)
+ %v0 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 0)
+ %v1 = call <vscale x 8 x i1> @llvm.vector.extract.nxv8i1.nxv16i1.i64(<vscale x 16 x i1> %r, i64 8)
+ tail call void @use(<vscale x 8 ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/202909
More information about the llvm-commits
mailing list