<div dir="ltr">It looks like you pushed this commit to the branch "master" -- which isn't in use anymore -- rather than "main".</div><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Thu, Feb 4, 2021 at 9:11 AM Joe Ellis via llvm-commits <<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex"><br>
Author: Joe Ellis<br>
Date: 2021-02-04T14:10:50Z<br>
New Revision: c9439ca36342fb6013187d0a69aef92736951476<br>
<br>
URL: <a href="https://github.com/llvm/llvm-project/commit/c9439ca36342fb6013187d0a69aef92736951476" rel="noreferrer" target="_blank">https://github.com/llvm/llvm-project/commit/c9439ca36342fb6013187d0a69aef92736951476</a><br>
DIFF: <a href="https://github.com/llvm/llvm-project/commit/c9439ca36342fb6013187d0a69aef92736951476.diff" rel="noreferrer" target="_blank">https://github.com/llvm/llvm-project/commit/c9439ca36342fb6013187d0a69aef92736951476.diff</a><br>
<br>
LOG: [AArch64][SVE] Coalesce calls to the SVE ptrue intrinsic where possible<br>
<br>
It is possible to eliminate redundant calls to the SVE ptrue intrinsic.<br>
For example: suppose that we have two SVE ptrue intrinsic calls P1 and<br>
P2. If P1 is at least as wide as P2, then P2 can be written as a<br>
reinterpret P1 using the SVE reinterpret intrinsics.<br>
<br>
Coalescing ptrue intrinsics can result in fewer ptrue instructions in<br>
the codegen, and is conducive to better analysis further down the line.<br>
<br>
This commit extends the aarch64-sve-intrinsic-opts pass to support<br>
coalescing ptrue intrisic calls.<br>
<br>
Differential Revision: <a href="https://reviews.llvm.org/D94230" rel="noreferrer" target="_blank">https://reviews.llvm.org/D94230</a><br>
<br>
Added: <br>
llvm/test/CodeGen/AArch64/sve-coalesce-ptrue-intrinsics.ll<br>
<br>
Modified: <br>
llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp<br>
<br>
Removed: <br>
<br>
<br>
<br>
################################################################################<br>
diff --git a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp<br>
index 9911f33371c6..3d9080f7997d 100644<br>
--- a/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp<br>
+++ b/llvm/lib/Target/AArch64/SVEIntrinsicOpts.cpp<br>
@@ -9,14 +9,20 @@<br>
//<br>
// Performs general IR level optimizations on SVE intrinsics.<br>
//<br>
-// The main goal of this pass is to remove unnecessary reinterpret<br>
-// intrinsics (llvm.aarch64.sve.convert.[to|from].svbool), e.g:<br>
+// This pass performs the following optimizations:<br>
//<br>
-// %1 = @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv4i1(<vscale x 4 x i1> %a)<br>
-// %2 = @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %1)<br>
+// - removes unnecessary reinterpret intrinsics<br>
+// (llvm.aarch64.sve.convert.[to|from].svbool), e.g:<br>
+// %1 = @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv4i1(<vscale x 4 x i1> %a)<br>
+// %2 = @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %1)<br>
//<br>
-// This pass also looks for ptest intrinsics & phi instructions where the<br>
-// operands are being needlessly converted to and from svbool_t.<br>
+// - removes unnecessary ptrue intrinsics (llvm.aarch64.sve.ptrue), e.g:<br>
+// %1 = @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)<br>
+// %2 = @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)<br>
+// ; (%1 can be replaced with a reinterpret of %2)<br>
+//<br>
+// - optimizes ptest intrinsics and phi instructions where the operands are<br>
+// being needlessly converted to and from svbool_t.<br>
//<br>
//===----------------------------------------------------------------------===//<br>
<br>
@@ -56,8 +62,17 @@ struct SVEIntrinsicOpts : public ModulePass {<br>
private:<br>
static IntrinsicInst *isReinterpretToSVBool(Value *V);<br>
<br>
+ bool coalescePTrueIntrinsicCalls(BasicBlock &BB,<br>
+ SmallSetVector<IntrinsicInst *, 4> &PTrues);<br>
+ bool optimizePTrueIntrinsicCalls(SmallSetVector<Function *, 4> &Functions);<br>
+<br>
+ /// Operates at the instruction-scope. I.e., optimizations are applied local<br>
+ /// to individual instructions.<br>
static bool optimizeIntrinsic(Instruction *I);<br>
+ bool optimizeIntrinsicCalls(SmallSetVector<Function *, 4> &Functions);<br>
<br>
+ /// Operates at the function-scope. I.e., optimizations are applied local to<br>
+ /// the functions themselves.<br>
bool optimizeFunctions(SmallSetVector<Function *, 4> &Functions);<br>
<br>
static bool optimizeConvertFromSVBool(IntrinsicInst *I);<br>
@@ -95,6 +110,188 @@ IntrinsicInst *SVEIntrinsicOpts::isReinterpretToSVBool(Value *V) {<br>
return I;<br>
}<br>
<br>
+/// Checks if a ptrue intrinsic call is promoted. The act of promoting a<br>
+/// ptrue will introduce zeroing. For example:<br>
+///<br>
+/// %1 = <vscale x 4 x i1> call @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)<br>
+/// %2 = <vscale x 16 x i1> call @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv4i1(<vscale x 4 x i1> %1)<br>
+/// %3 = <vscale x 8 x i1> call @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> %2)<br>
+///<br>
+/// %1 is promoted, because it is converted:<br>
+///<br>
+/// <vscale x 4 x i1> => <vscale x 16 x i1> => <vscale x 8 x i1><br>
+///<br>
+/// via a sequence of the SVE reinterpret intrinsics convert.{to,from}.svbool.<br>
+bool isPTruePromoted(IntrinsicInst *PTrue) {<br>
+ // Find all users of this intrinsic that are calls to convert-to-svbool<br>
+ // reinterpret intrinsics.<br>
+ SmallVector<IntrinsicInst *, 4> ConvertToUses;<br>
+ for (User *User : PTrue->users()) {<br>
+ if (match(User, m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>())) {<br>
+ ConvertToUses.push_back(cast<IntrinsicInst>(User));<br>
+ }<br>
+ }<br>
+<br>
+ // If no such calls were found, this is ptrue is not promoted.<br>
+ if (ConvertToUses.empty())<br>
+ return false;<br>
+<br>
+ // Otherwise, try to find users of the convert-to-svbool intrinsics that are<br>
+ // calls to the convert-from-svbool intrinsic, and would result in some lanes<br>
+ // being zeroed.<br>
+ const auto *PTrueVTy = cast<ScalableVectorType>(PTrue->getType());<br>
+ for (IntrinsicInst *ConvertToUse : ConvertToUses) {<br>
+ for (User *User : ConvertToUse->users()) {<br>
+ auto *IntrUser = dyn_cast<IntrinsicInst>(User);<br>
+ if (IntrUser && IntrUser->getIntrinsicID() ==<br>
+ Intrinsic::aarch64_sve_convert_from_svbool) {<br>
+ const auto *IntrUserVTy = cast<ScalableVectorType>(IntrUser->getType());<br>
+<br>
+ // Would some lanes become zeroed by the conversion?<br>
+ if (IntrUserVTy->getElementCount().getKnownMinValue() ><br>
+ PTrueVTy->getElementCount().getKnownMinValue())<br>
+ // This is a promoted ptrue.<br>
+ return true;<br>
+ }<br>
+ }<br>
+ }<br>
+<br>
+ // If no matching calls were found, this is not a promoted ptrue.<br>
+ return false;<br>
+}<br>
+<br>
+/// Attempts to coalesce ptrues in a basic block.<br>
+bool SVEIntrinsicOpts::coalescePTrueIntrinsicCalls(<br>
+ BasicBlock &BB, SmallSetVector<IntrinsicInst *, 4> &PTrues) {<br>
+ if (PTrues.size() <= 1)<br>
+ return false;<br>
+<br>
+ // Find the ptrue with the most lanes.<br>
+ auto *MostEncompassingPTrue = *std::max_element(<br>
+ PTrues.begin(), PTrues.end(), [](auto *PTrue1, auto *PTrue2) {<br>
+ auto *PTrue1VTy = cast<ScalableVectorType>(PTrue1->getType());<br>
+ auto *PTrue2VTy = cast<ScalableVectorType>(PTrue2->getType());<br>
+ return PTrue1VTy->getElementCount().getKnownMinValue() <<br>
+ PTrue2VTy->getElementCount().getKnownMinValue();<br>
+ });<br>
+<br>
+ // Remove the most encompassing ptrue, as well as any promoted ptrues, leaving<br>
+ // behind only the ptrues to be coalesced.<br>
+ PTrues.remove(MostEncompassingPTrue);<br>
+ PTrues.remove_if([](auto *PTrue) { return isPTruePromoted(PTrue); });<br>
+<br>
+ // Hoist MostEncompassingPTrue to the start of the basic block. It is always<br>
+ // safe to do this, since ptrue intrinsic calls are guaranteed to have no<br>
+ // predecessors.<br>
+ MostEncompassingPTrue->moveBefore(BB, BB.getFirstInsertionPt());<br>
+<br>
+ LLVMContext &Ctx = BB.getContext();<br>
+ IRBuilder<> Builder(Ctx);<br>
+ Builder.SetInsertPoint(&BB, ++MostEncompassingPTrue->getIterator());<br>
+<br>
+ auto *MostEncompassingPTrueVTy =<br>
+ cast<VectorType>(MostEncompassingPTrue->getType());<br>
+ auto *ConvertToSVBool = Builder.CreateIntrinsic(<br>
+ Intrinsic::aarch64_sve_convert_to_svbool, {MostEncompassingPTrueVTy},<br>
+ {MostEncompassingPTrue});<br>
+<br>
+ for (auto *PTrue : PTrues) {<br>
+ auto *PTrueVTy = cast<VectorType>(PTrue->getType());<br>
+<br>
+ Builder.SetInsertPoint(&BB, ++ConvertToSVBool->getIterator());<br>
+ auto *ConvertFromSVBool =<br>
+ Builder.CreateIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool,<br>
+ {PTrueVTy}, {ConvertToSVBool});<br>
+ PTrue->replaceAllUsesWith(ConvertFromSVBool);<br>
+ PTrue->eraseFromParent();<br>
+ }<br>
+<br>
+ return true;<br>
+}<br>
+<br>
+/// The goal of this function is to remove redundant calls to the SVE ptrue<br>
+/// intrinsic in each basic block within the given functions.<br>
+///<br>
+/// SVE ptrues have two representations in LLVM IR:<br>
+/// - a logical representation -- an arbitrary-width scalable vector of i1s,<br>
+/// i.e. <vscale x N x i1>.<br>
+/// - a physical representation (svbool, <vscale x 16 x i1>) -- a 16-element<br>
+/// scalable vector of i1s, i.e. <vscale x 16 x i1>.<br>
+///<br>
+/// The SVE ptrue intrinsic is used to create a logical representation of an SVE<br>
+/// predicate. Suppose that we have two SVE ptrue intrinsic calls: P1 and P2. If<br>
+/// P1 creates a logical SVE predicate that is at least as wide as the logical<br>
+/// SVE predicate created by P2, then all of the bits that are true in the<br>
+/// physical representation of P2 are necessarily also true in the physical<br>
+/// representation of P1. P1 'encompasses' P2, therefore, the intrinsic call to<br>
+/// P2 is redundant and can be replaced by an SVE reinterpret of P1 via<br>
+/// convert.{to,from}.svbool.<br>
+///<br>
+/// Currently, this pass only coalesces calls to SVE ptrue intrinsics<br>
+/// if they match the following conditions:<br>
+///<br>
+/// - the call to the intrinsic uses either the SV_ALL or SV_POW2 patterns.<br>
+/// SV_ALL indicates that all bits of the predicate vector are to be set to<br>
+/// true. SV_POW2 indicates that all bits of the predicate vector up to the<br>
+/// largest power-of-two are to be set to true.<br>
+/// - the result of the call to the intrinsic is not promoted to a wider<br>
+/// predicate. In this case, keeping the extra ptrue leads to better codegen<br>
+/// -- coalescing here would create an irreducible chain of SVE reinterprets<br>
+/// via convert.{to,from}.svbool.<br>
+///<br>
+/// EXAMPLE:<br>
+///<br>
+/// %1 = <vscale x 8 x i1> ptrue(i32 SV_ALL)<br>
+/// ; Logical: <1, 1, 1, 1, 1, 1, 1, 1><br>
+/// ; Physical: <1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0><br>
+/// ...<br>
+///<br>
+/// %2 = <vscale x 4 x i1> ptrue(i32 SV_ALL)<br>
+/// ; Logical: <1, 1, 1, 1><br>
+/// ; Physical: <1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0><br>
+/// ...<br>
+///<br>
+/// Here, %2 can be replaced by an SVE reinterpret of %1, giving, for instance:<br>
+///<br>
+/// %1 = <vscale x 8 x i1> ptrue(i32 i31)<br>
+/// %2 = <vscale x 16 x i1> convert.to.svbool(<vscale x 8 x i1> %1)<br>
+/// %3 = <vscale x 4 x i1> convert.from.svbool(<vscale x 16 x i1> %2)<br>
+///<br>
+bool SVEIntrinsicOpts::optimizePTrueIntrinsicCalls(<br>
+ SmallSetVector<Function *, 4> &Functions) {<br>
+ bool Changed = false;<br>
+<br>
+ for (auto *F : Functions) {<br>
+ for (auto &BB : *F) {<br>
+ SmallSetVector<IntrinsicInst *, 4> SVAllPTrues;<br>
+ SmallSetVector<IntrinsicInst *, 4> SVPow2PTrues;<br>
+<br>
+ // For each basic block, collect the used ptrues and try to coalesce them.<br>
+ for (Instruction &I : BB) {<br>
+ if (I.use_empty())<br>
+ continue;<br>
+<br>
+ auto *IntrI = dyn_cast<IntrinsicInst>(&I);<br>
+ if (!IntrI || IntrI->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue)<br>
+ continue;<br>
+<br>
+ const auto PTruePattern =<br>
+ cast<ConstantInt>(IntrI->getOperand(0))->getZExtValue();<br>
+<br>
+ if (PTruePattern == AArch64SVEPredPattern::all)<br>
+ SVAllPTrues.insert(IntrI);<br>
+ if (PTruePattern == AArch64SVEPredPattern::pow2)<br>
+ SVPow2PTrues.insert(IntrI);<br>
+ }<br>
+<br>
+ Changed |= coalescePTrueIntrinsicCalls(BB, SVAllPTrues);<br>
+ Changed |= coalescePTrueIntrinsicCalls(BB, SVPow2PTrues);<br>
+ }<br>
+ }<br>
+<br>
+ return Changed;<br>
+}<br>
+<br>
/// The function will remove redundant reinterprets casting in the presence<br>
/// of the control flow<br>
bool SVEIntrinsicOpts::processPhiNode(IntrinsicInst *X) {<br>
@@ -243,7 +440,7 @@ bool SVEIntrinsicOpts::optimizeIntrinsic(Instruction *I) {<br>
return true;<br>
}<br>
<br>
-bool SVEIntrinsicOpts::optimizeFunctions(<br>
+bool SVEIntrinsicOpts::optimizeIntrinsicCalls(<br>
SmallSetVector<Function *, 4> &Functions) {<br>
bool Changed = false;<br>
for (auto *F : Functions) {<br>
@@ -260,6 +457,16 @@ bool SVEIntrinsicOpts::optimizeFunctions(<br>
return Changed;<br>
}<br>
<br>
+bool SVEIntrinsicOpts::optimizeFunctions(<br>
+ SmallSetVector<Function *, 4> &Functions) {<br>
+ bool Changed = false;<br>
+<br>
+ Changed |= optimizePTrueIntrinsicCalls(Functions);<br>
+ Changed |= optimizeIntrinsicCalls(Functions);<br>
+<br>
+ return Changed;<br>
+}<br>
+<br>
bool SVEIntrinsicOpts::runOnModule(Module &M) {<br>
bool Changed = false;<br>
SmallSetVector<Function *, 4> Functions;<br>
@@ -276,6 +483,7 @@ bool SVEIntrinsicOpts::runOnModule(Module &M) {<br>
case Intrinsic::aarch64_sve_ptest_any:<br>
case Intrinsic::aarch64_sve_ptest_first:<br>
case Intrinsic::aarch64_sve_ptest_last:<br>
+ case Intrinsic::aarch64_sve_ptrue:<br>
for (User *U : F.users())<br>
Functions.insert(cast<Instruction>(U)->getFunction());<br>
break;<br>
<br>
diff --git a/llvm/test/CodeGen/AArch64/sve-coalesce-ptrue-intrinsics.ll b/llvm/test/CodeGen/AArch64/sve-coalesce-ptrue-intrinsics.ll<br>
new file mode 100644<br>
index 000000000000..526f7bc52f59<br>
--- /dev/null<br>
+++ b/llvm/test/CodeGen/AArch64/sve-coalesce-ptrue-intrinsics.ll<br>
@@ -0,0 +1,189 @@<br>
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py<br>
+; RUN: opt -S -aarch64-sve-intrinsic-opts -mtriple=aarch64-linux-gnu -mattr=+sve < %s 2>%t | FileCheck %s<br>
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t<br>
+<br>
+; If this check fails please read test/CodeGen/AArch64/README for instructions on how to resolve it.<br>
+; WARN-NOT: warning<br>
+<br>
+declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 immarg)<br>
+declare <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 immarg)<br>
+declare <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 immarg)<br>
+declare <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 immarg)<br>
+<br>
+declare <vscale x 16 x i32> @llvm.aarch64.sve.ld1.nxv16i32(<vscale x 16 x i1>, i32*)<br>
+declare <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1>, i32*)<br>
+declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1>, i32*)<br>
+declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1>, i16*)<br>
+declare <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1>, i32*)<br>
+<br>
+declare <vscale x 16 x i1> @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv4i1(<vscale x 4 x i1>)<br>
+declare <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1>)<br>
+<br>
+; Two calls to the SVE ptrue intrinsic. %1 is redundant, and can be expressed as an SVE reinterpret of %3 via<br>
+; convert.{to,from}.svbool.<br>
+define <vscale x 8 x i32> @coalesce_test_basic(i32* %addr) {<br>
+; CHECK-LABEL: @coalesce_test_basic(<br>
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)<br>
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i1> @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv8i1(<vscale x 8 x i1> [[TMP1]])<br>
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[TMP2]])<br>
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> [[TMP3]], i32* [[ADDR:%.*]])<br>
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> [[TMP1]], i32* [[ADDR]])<br>
+; CHECK-NEXT: ret <vscale x 8 x i32> [[TMP5]]<br>
+;<br>
+ %1 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)<br>
+ %2 = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %1, i32* %addr)<br>
+ %3 = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)<br>
+ %4 = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> %3, i32* %addr)<br>
+ ret <vscale x 8 x i32> %4<br>
+}<br>
+<br>
+; Two calls to the SVE ptrue intrinsic with the SV_POW2 pattern. This should reduce to the same output as<br>
+; coalesce_test_basic.<br>
+define <vscale x 8 x i32> @coalesce_test_pow2(i32* %addr) {<br>
+; CHECK-LABEL: @coalesce_test_pow2(<br>
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 0)<br>
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i1> @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv8i1(<vscale x 8 x i1> [[TMP1]])<br>
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[TMP2]])<br>
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> [[TMP3]], i32* [[ADDR:%.*]])<br>
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> [[TMP1]], i32* [[ADDR]])<br>
+; CHECK-NEXT: ret <vscale x 8 x i32> [[TMP5]]<br>
+;<br>
+ %1 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 0)<br>
+ %2 = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %1, i32* %addr)<br>
+ %3 = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 0)<br>
+ %4 = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> %3, i32* %addr)<br>
+ ret <vscale x 8 x i32> %4<br>
+}<br>
+<br>
+; Four calls to the SVE ptrue intrinsic; two with the SV_ALL patterns, and two with the SV_POW2 pattern. The<br>
+; two SV_ALL ptrue intrinsics should be coalesced, and the two SV_POW2 intrinsics should be colaesced.<br>
+define <vscale x 8 x i32> @coalesce_test_all_and_pow2(i32* %addr) {<br>
+; CHECK-LABEL: @coalesce_test_all_and_pow2(<br>
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 0)<br>
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i1> @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv8i1(<vscale x 8 x i1> [[TMP1]])<br>
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[TMP2]])<br>
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)<br>
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 16 x i1> @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv8i1(<vscale x 8 x i1> [[TMP4]])<br>
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[TMP5]])<br>
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> [[TMP3]], i32* [[ADDR:%.*]])<br>
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> [[TMP1]], i32* [[ADDR]])<br>
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> [[TMP6]], i32* [[ADDR]])<br>
+; CHECK-NEXT: [[TMP10:%.*]] = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> [[TMP4]], i32* [[ADDR]])<br>
+; CHECK-NEXT: ret <vscale x 8 x i32> [[TMP10]]<br>
+;<br>
+ %1 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 0)<br>
+ %2 = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 0)<br>
+ %3 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)<br>
+ %4 = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)<br>
+<br>
+ %5 = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %1, i32* %addr)<br>
+ %6 = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> %2, i32* %addr)<br>
+ %7 = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %3, i32* %addr)<br>
+ %8 = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> %4, i32* %addr)<br>
+ ret <vscale x 8 x i32> %8<br>
+}<br>
+<br>
+<br>
+; Two calls to the SVE ptrue intrinsic: one with the SV_ALL pattern, another with the SV_POW2 pattern. The<br>
+; patterns are incompatible, so they should not be coalesced.<br>
+define <vscale x 8 x i32> @coalesce_test_pattern_mismatch2(i32* %addr) {<br>
+; CHECK-LABEL: @coalesce_test_pattern_mismatch2(<br>
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 0)<br>
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> [[TMP1]], i32* [[ADDR:%.*]])<br>
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)<br>
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> [[TMP3]], i32* [[ADDR]])<br>
+; CHECK-NEXT: ret <vscale x 8 x i32> [[TMP4]]<br>
+;<br>
+ %1 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 0)<br>
+ %2 = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %1, i32* %addr)<br>
+ %3 = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)<br>
+ %4 = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> %3, i32* %addr)<br>
+ ret <vscale x 8 x i32> %4<br>
+}<br>
+<br>
+; Two calls to the SVE ptrue intrinsic with the SV_VL1 pattern. This pattern is not currently recognised, so<br>
+; nothing should be done here.<br>
+define <vscale x 8 x i32> @coalesce_test_bad_pattern(i32* %addr) {<br>
+; CHECK-LABEL: @coalesce_test_bad_pattern(<br>
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 1)<br>
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> [[TMP1]], i32* [[ADDR:%.*]])<br>
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 1)<br>
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> [[TMP3]], i32* [[ADDR]])<br>
+; CHECK-NEXT: ret <vscale x 8 x i32> [[TMP4]]<br>
+;<br>
+ %1 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 1)<br>
+ %2 = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %1, i32* %addr)<br>
+ %3 = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 1)<br>
+ %4 = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> %3, i32* %addr)<br>
+ ret <vscale x 8 x i32> %4<br>
+}<br>
+<br>
+; Four calls to the SVE ptrue intrinsic. %7 is the most encompassing, and the others can be expressed as an<br>
+; SVE reinterprets of %7 via convert.{to,from}.svbool.<br>
+define <vscale x 16 x i32> @coalesce_test_multiple(i32* %addr) {<br>
+; CHECK-LABEL: @coalesce_test_multiple(<br>
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)<br>
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i1> @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv16i1(<vscale x 16 x i1> [[TMP1]])<br>
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[TMP2]])<br>
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[TMP2]])<br>
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[TMP2]])<br>
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1> [[TMP5]], i32* [[ADDR:%.*]])<br>
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> [[TMP4]], i32* [[ADDR]])<br>
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> [[TMP3]], i32* [[ADDR]])<br>
+; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 16 x i32> @llvm.aarch64.sve.ld1.nxv16i32(<vscale x 16 x i1> [[TMP1]], i32* [[ADDR]])<br>
+; CHECK-NEXT: ret <vscale x 16 x i32> [[TMP9]]<br>
+;<br>
+ %1 = tail call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)<br>
+ %2 = call <vscale x 2 x i32> @llvm.aarch64.sve.ld1.nxv2i32(<vscale x 2 x i1> %1, i32* %addr)<br>
+ %3 = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)<br>
+ %4 = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %3, i32* %addr)<br>
+ %5 = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)<br>
+ %6 = call <vscale x 8 x i32> @llvm.aarch64.sve.ld1.nxv8i32(<vscale x 8 x i1> %5, i32* %addr)<br>
+ %7 = tail call <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 31)<br>
+ %8 = call <vscale x 16 x i32> @llvm.aarch64.sve.ld1.nxv16i32(<vscale x 16 x i1> %7, i32* %addr)<br>
+ ret <vscale x 16 x i32> %8<br>
+}<br>
+<br>
+; Two calls to the SVE ptrue intrinsic which are both of the same size. In this case, one should be identified<br>
+; as redundant and rewritten and an SVE reinterpret of the other via the convert.{to,from}.svbool intrinsics.<br>
+; This introduces a redundant conversion which will then be eliminated.<br>
+define <vscale x 4 x i32> @coalesce_test_same_size(i32* %addr) {<br>
+; CHECK-LABEL: @coalesce_test_same_size(<br>
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)<br>
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> [[TMP1]], i32* [[ADDR:%.*]])<br>
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> [[TMP1]], i32* [[ADDR]])<br>
+; CHECK-NEXT: ret <vscale x 4 x i32> [[TMP3]]<br>
+;<br>
+ %1 = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)<br>
+ %2 = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %1, i32* %addr)<br>
+ %3 = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)<br>
+ %4 = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %3, i32* %addr)<br>
+ ret <vscale x 4 x i32> %4<br>
+}<br>
+<br>
+; Two calls to the SVE ptrue intrinsic, but neither can be eliminated; %1 is promoted to become %3, which<br>
+; means eliminating this call to the SVE ptrue intrinsic would involve creating a longer, irreducible chain of<br>
+; conversions. Better codegen is achieved by just leaving the ptrue as-is.<br>
+define <vscale x 8 x i16> @coalesce_test_promoted_ptrue(i32* %addr1, i16* %addr2) {<br>
+; CHECK-LABEL: @coalesce_test_promoted_ptrue(<br>
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)<br>
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 16 x i1> @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv8i1(<vscale x 8 x i1> [[TMP1]])<br>
+; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)<br>
+; CHECK-NEXT: [[TMP4:%.*]] = call <vscale x 16 x i1> @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv4i1(<vscale x 4 x i1> [[TMP3]])<br>
+; CHECK-NEXT: [[TMP5:%.*]] = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[TMP4]])<br>
+; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> [[TMP3]], i32* [[ADDR1:%.*]])<br>
+; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1> [[TMP5]], i16* [[ADDR2:%.*]])<br>
+; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1> [[TMP1]], i16* [[ADDR2]])<br>
+; CHECK-NEXT: ret <vscale x 8 x i16> [[TMP8]]<br>
+;<br>
+ %1 = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)<br>
+ %2 = call <vscale x 16 x i1> @<a href="http://llvm.aarch64.sve.convert.to" target="_blank">llvm.aarch64.sve.convert.to</a>.svbool.nxv4i1(<vscale x 4 x i1> %1)<br>
+ %3 = call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %2)<br>
+<br>
+ %4 = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1.nxv4i32(<vscale x 4 x i1> %1, i32* %addr1)<br>
+ %5 = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1> %3, i16* %addr2)<br>
+<br>
+ %6 = call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)<br>
+ %7 = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1.nxv8i16(<vscale x 8 x i1> %6, i16* %addr2)<br>
+ ret <vscale x 8 x i16> %7<br>
+}<br>
<br>
<br>
<br>
_______________________________________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@lists.llvm.org" target="_blank">llvm-commits@lists.llvm.org</a><br>
<a href="https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" target="_blank">https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits</a><br>
</blockquote></div>