[polly] [Polly] Add vectorize metadata to loops identified as vectorizable by polly (PR #113994)
Karthika Devi C via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 18 09:11:55 PST 2024
https://github.com/kartcq updated https://github.com/llvm/llvm-project/pull/113994
>From 337ef3625408826e3359be89d36aa5062bd22088 Mon Sep 17 00:00:00 2001
From: Karthika Devi C <quic_kartc at quicinc.com>
Date: Mon, 22 Apr 2024 03:18:08 -0700
Subject: [PATCH 1/8] [Polly] Add vectorize metadata to loops identified as
vectorizable by polly
This patch introduces the initial implementation for annotating
loops created by Polly. Polly generates RunTimeChecks (RTCs), which
result in loop versioning. Specifically, the loop created by Polly
is executed when the RTCs pass, otherwise, the original loop is executed.
This patch adds the "llvm.loop.vectorize.enable" metadata, setting
it to true for loops created by Polly. It also disables vectorization
for the original fallback loop.
This behavior is controlled by the 'polly-annotate-metadata-vectorize'
flag, and the annotations are applied only when this flag is enabled.
This flag is set to false by default.
NOTE: This commit is initial patch in effort to make polly interact with
Loop Vectorizer via metadata.
---
polly/include/polly/CodeGen/IRBuilder.h | 3 +-
polly/lib/CodeGen/CodeGeneration.cpp | 25 ++++++++
polly/lib/CodeGen/IRBuilder.cpp | 36 ++++++-----
polly/lib/CodeGen/LoopGenerators.cpp | 18 +++++-
.../CodeGen/Metadata/basic_vec_annotate.ll | 61 +++++++++++++++++++
5 files changed, 126 insertions(+), 17 deletions(-)
create mode 100644 polly/test/CodeGen/Metadata/basic_vec_annotate.ll
diff --git a/polly/include/polly/CodeGen/IRBuilder.h b/polly/include/polly/CodeGen/IRBuilder.h
index ffca887fbc09aa..73571dbcf1f2bd 100644
--- a/polly/include/polly/CodeGen/IRBuilder.h
+++ b/polly/include/polly/CodeGen/IRBuilder.h
@@ -60,7 +60,8 @@ class ScopAnnotator {
/// Annotate the loop latch @p B wrt. @p L.
void annotateLoopLatch(llvm::BranchInst *B, llvm::Loop *L, bool IsParallel,
- bool IsLoopVectorizerDisabled) const;
+ bool setVectorizeMetadata,
+ bool EnableLoopVectorizer) const;
/// Add alternative alias based pointers
///
diff --git a/polly/lib/CodeGen/CodeGeneration.cpp b/polly/lib/CodeGen/CodeGeneration.cpp
index 8813cfd959ef6e..bfc4014c848d5b 100644
--- a/polly/lib/CodeGen/CodeGeneration.cpp
+++ b/polly/lib/CodeGen/CodeGeneration.cpp
@@ -54,6 +54,11 @@ static cl::opt<bool> Verify("polly-codegen-verify",
cl::desc("Verify the function generated by Polly"),
cl::Hidden, cl::cat(PollyCategory));
+cl::opt<bool> PollyVectorizeMetadata(
+ "polly-annotate-metadata-vectorize"
+ cl::desc("Append vectorize enable/disable metadata from polly"),
+ cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
+
bool polly::PerfMonitoring;
static cl::opt<bool, true>
@@ -233,6 +238,26 @@ static bool generateCode(Scop &S, IslAstInfo &AI, LoopInfo &LI,
NodeBuilder.allocateNewArrays(StartExitBlocks);
Annotator.buildAliasScopes(S);
+ // The code below annotates the "llvm.loop.vectorize.enable" to false
+ // for the code flow taken when RTCs fail. Because we don't want the
+ // Loop Vectorizer to come in later and vectorize the original fall back
+ // loop when 'polly-annotate-metadata-vectorize' is passed.
+ if (PollyVectorizeMetadata && &Annotator) {
+ for (Loop *L : LI.getLoopsInPreorder()) {
+ if (S.contains(L)) {
+ Annotator.pushLoop(L, false);
+ SmallVector<BasicBlock *, 4> LoopLatchBlocks;
+ L->getLoopLatches(LoopLatchBlocks);
+ for (BasicBlock *ControlBB : LoopLatchBlocks) {
+ BranchInst *Br = dyn_cast<BranchInst>(ControlBB->getTerminator());
+ if (Br)
+ Annotator.annotateLoopLatch(Br, L, false, true, false);
+ }
+ Annotator.popLoop(false);
+ }
+ }
+ }
+
if (PerfMonitoring) {
PerfMonitor P(S, EnteringBB->getParent()->getParent());
P.initialize();
diff --git a/polly/lib/CodeGen/IRBuilder.cpp b/polly/lib/CodeGen/IRBuilder.cpp
index 2285c746912f4e..4e01b86da563a8 100644
--- a/polly/lib/CodeGen/IRBuilder.cpp
+++ b/polly/lib/CodeGen/IRBuilder.cpp
@@ -128,8 +128,25 @@ void ScopAnnotator::popLoop(bool IsParallel) {
LoopAttrEnv.pop_back();
}
+void addVectorizeMetadata(LLVMContext &Ctx, SmallVector<Metadata *, 3> *Args,
+ bool EnableLoopVectorizer) {
+ MDString *PropName = MDString::get(Ctx, "llvm.loop.vectorize.enable");
+ ConstantInt *Value =
+ ConstantInt::get(Type::getInt1Ty(Ctx), EnableLoopVectorizer);
+ ValueAsMetadata *PropValue = ValueAsMetadata::get(Value);
+ Args->push_back(MDNode::get(Ctx, {PropName, PropValue}));
+}
+
+void addParallelMetadata(LLVMContext &Ctx, SmallVector<Metadata *, 3> *Args,
+ llvm::SmallVector<llvm::MDNode *, 8> ParallelLoops) {
+ MDString *PropName = MDString::get(Ctx, "llvm.loop.parallel_accesses");
+ MDNode *AccGroup = ParallelLoops.back();
+ Args->push_back(MDNode::get(Ctx, {PropName, AccGroup}));
+}
+
void ScopAnnotator::annotateLoopLatch(BranchInst *B, Loop *L, bool IsParallel,
- bool IsLoopVectorizerDisabled) const {
+ bool setVectorizeMetadata,
+ bool EnableLoopVectorizer) const {
LLVMContext &Ctx = SE->getContext();
SmallVector<Metadata *, 3> Args;
@@ -145,19 +162,10 @@ void ScopAnnotator::annotateLoopLatch(BranchInst *B, Loop *L, bool IsParallel,
if (MData)
llvm::append_range(Args, drop_begin(MData->operands(), 1));
}
-
- if (IsLoopVectorizerDisabled) {
- MDString *PropName = MDString::get(Ctx, "llvm.loop.vectorize.enable");
- ConstantInt *FalseValue = ConstantInt::get(Type::getInt1Ty(Ctx), 0);
- ValueAsMetadata *PropValue = ValueAsMetadata::get(FalseValue);
- Args.push_back(MDNode::get(Ctx, {PropName, PropValue}));
- }
-
- if (IsParallel) {
- MDString *PropName = MDString::get(Ctx, "llvm.loop.parallel_accesses");
- MDNode *AccGroup = ParallelLoops.back();
- Args.push_back(MDNode::get(Ctx, {PropName, AccGroup}));
- }
+ if (IsParallel)
+ addParallelMetadata(Ctx, &Args, ParallelLoops);
+ if (setVectorizeMetadata)
+ addVectorizeMetadata(Ctx, &Args, EnableLoopVectorizer);
// No metadata to annotate.
if (!MData && Args.size() <= 1)
diff --git a/polly/lib/CodeGen/LoopGenerators.cpp b/polly/lib/CodeGen/LoopGenerators.cpp
index b4f8bb8948c282..c082b61096250d 100644
--- a/polly/lib/CodeGen/LoopGenerators.cpp
+++ b/polly/lib/CodeGen/LoopGenerators.cpp
@@ -14,6 +14,7 @@
#include "polly/CodeGen/LoopGenerators.h"
#include "polly/Options.h"
#include "polly/ScopDetection.h"
+#include "polly/ScopInfo.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfoMetadata.h"
@@ -35,6 +36,8 @@ static cl::opt<int, true>
cl::Hidden, cl::location(polly::PollyNumThreads),
cl::init(0), cl::cat(PollyCategory));
+extern cl::opt<bool> PollyVectorizeMetadata;
+
static cl::opt<OMPGeneralSchedulingType, true> XPollyScheduling(
"polly-scheduling",
cl::desc("Scheduling type of parallel OpenMP for loops"),
@@ -159,8 +162,19 @@ Value *polly::createLoop(Value *LB, Value *UB, Value *Stride,
// Create the loop latch and annotate it as such.
BranchInst *B = Builder.CreateCondBr(LoopCondition, HeaderBB, ExitBB);
- if (Annotator)
- Annotator->annotateLoopLatch(B, NewLoop, Parallel, LoopVectDisabled);
+
+ // If the 'polly-annotate-metadata-vectorize' flag is passed, we add
+ // the vectorize metadata. Otherwise we fall back to previous behavior
+ // of annotating the loop only when LoopVectDisabled is true.
+ if (Annotator) {
+ if (PollyVectorizeMetadata)
+ Annotator->annotateLoopLatch(B, NewLoop, Parallel, true,
+ !LoopVectDisabled);
+ else if (LoopVectDisabled)
+ Annotator->annotateLoopLatch(B, NewLoop, Parallel, true, false);
+ else
+ Annotator->annotateLoopLatch(B, NewLoop, Parallel, false, false);
+ }
IV->addIncoming(IncrementedIV, HeaderBB);
if (GuardBB)
diff --git a/polly/test/CodeGen/Metadata/basic_vec_annotate.ll b/polly/test/CodeGen/Metadata/basic_vec_annotate.ll
new file mode 100644
index 00000000000000..b4b1a5c8160264
--- /dev/null
+++ b/polly/test/CodeGen/Metadata/basic_vec_annotate.ll
@@ -0,0 +1,61 @@
+; RUN: opt %loadNPMPolly -S -passes=polly-codegen -polly-annotate-metadata-vectorize < %s | FileCheck %s
+
+; Basic verification of vectorize metadata getting added when "-polly-vectorize-metadata" is
+; passed.
+
+; void add(int *A, int *B, int *C,int n) {
+; for(int i=0; i<n; i++)
+; C[i] += A[i] + B[i];
+; }
+
+; CHECK: for.body:
+; CHECK: br {{.*}} !llvm.loop [[LOOP:![0-9]+]]
+; CHECK: polly.stmt.for.body:
+; CHECK: br {{.*}} !llvm.loop [[POLLY_LOOP:![0-9]+]]
+; CHECK: [[LOOP]] = distinct !{[[LOOP]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"llvm.loop.vectorize.enable", i1 false}
+; CHECK: [[POLLY_LOOP]] = distinct !{[[POLLY_LOOP]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK: [[META3]] = !{!"llvm.loop.vectorize.enable", i1 true}
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+target triple = "aarch64-unknown-linux-gnu"
+
+; Function Attrs: nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable
+define dso_local void @add(ptr nocapture noundef readonly %A, ptr nocapture noundef readonly %B, ptr nocapture noundef %C, i32 noundef %n) local_unnamed_addr #0 {
+entry:
+ br label %entry.split
+
+entry.split: ; preds = %entry
+ %cmp10 = icmp sgt i32 %n, 0
+ br i1 %cmp10, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader: ; preds = %entry.split
+ %wide.trip.count = zext nneg i32 %n to i64
+ br label %for.body
+
+for.cond.cleanup.loopexit: ; preds = %for.body
+ br label %for.cond.cleanup
+
+for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry.split
+ ret void
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+ %arrayidx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv
+ %0 = load i32, ptr %arrayidx, align 4
+ %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv
+ %1 = load i32, ptr %arrayidx2, align 4
+ %add = add nsw i32 %1, %0
+ %arrayidx4 = getelementptr inbounds i32, ptr %C, i64 %indvars.iv
+ %2 = load i32, ptr %arrayidx4, align 4
+ %add5 = add nsw i32 %add, %2
+ store i32 %add5, ptr %arrayidx4, align 4
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body, !llvm.loop !0
+}
+
+attributes #0 = { nofree norecurse nosync nounwind memory(argmem: readwrite) uwtable "frame-pointer"="non-leaf" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-a57" "target-features"="+aes,+crc,+fp-armv8,+neon,+outline-atomics,+perfmon,+sha2,+v8a,-fmv" }
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.mustprogress"}
>From 374788e3afe28562530fd55fd7b5554b94a8d20b Mon Sep 17 00:00:00 2001
From: Karthika Devi C <quic_kartc at quicinc.com>
Date: Tue, 29 Oct 2024 07:38:30 +0530
Subject: [PATCH 2/8] Update CodeGeneration.cpp
---
polly/lib/CodeGen/CodeGeneration.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/polly/lib/CodeGen/CodeGeneration.cpp b/polly/lib/CodeGen/CodeGeneration.cpp
index bfc4014c848d5b..be9364da0fa08b 100644
--- a/polly/lib/CodeGen/CodeGeneration.cpp
+++ b/polly/lib/CodeGen/CodeGeneration.cpp
@@ -55,7 +55,7 @@ static cl::opt<bool> Verify("polly-codegen-verify",
cl::Hidden, cl::cat(PollyCategory));
cl::opt<bool> PollyVectorizeMetadata(
- "polly-annotate-metadata-vectorize"
+ "polly-annotate-metadata-vectorize",
cl::desc("Append vectorize enable/disable metadata from polly"),
cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
>From fd61d16e652a1308326ea9d203c476abf6da19bd Mon Sep 17 00:00:00 2001
From: Karthika Devi C <quic_kartc at quicinc.com>
Date: Wed, 30 Oct 2024 10:27:26 +0530
Subject: [PATCH 3/8] Apply suggestions from code review
Co-authored-by: Michael Kruse <github at meinersbur.de>
---
polly/include/polly/CodeGen/IRBuilder.h | 2 +-
polly/lib/CodeGen/CodeGeneration.cpp | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/polly/include/polly/CodeGen/IRBuilder.h b/polly/include/polly/CodeGen/IRBuilder.h
index 73571dbcf1f2bd..a45abe121b81ac 100644
--- a/polly/include/polly/CodeGen/IRBuilder.h
+++ b/polly/include/polly/CodeGen/IRBuilder.h
@@ -60,7 +60,7 @@ class ScopAnnotator {
/// Annotate the loop latch @p B wrt. @p L.
void annotateLoopLatch(llvm::BranchInst *B, llvm::Loop *L, bool IsParallel,
- bool setVectorizeMetadata,
+ bool SetVectorizeMetadata,
bool EnableLoopVectorizer) const;
/// Add alternative alias based pointers
diff --git a/polly/lib/CodeGen/CodeGeneration.cpp b/polly/lib/CodeGen/CodeGeneration.cpp
index be9364da0fa08b..18b29f0998afd9 100644
--- a/polly/lib/CodeGen/CodeGeneration.cpp
+++ b/polly/lib/CodeGen/CodeGeneration.cpp
@@ -242,11 +242,11 @@ static bool generateCode(Scop &S, IslAstInfo &AI, LoopInfo &LI,
// for the code flow taken when RTCs fail. Because we don't want the
// Loop Vectorizer to come in later and vectorize the original fall back
// loop when 'polly-annotate-metadata-vectorize' is passed.
- if (PollyVectorizeMetadata && &Annotator) {
+ if (PollyVectorizeMetadata) {
for (Loop *L : LI.getLoopsInPreorder()) {
if (S.contains(L)) {
Annotator.pushLoop(L, false);
- SmallVector<BasicBlock *, 4> LoopLatchBlocks;
+ SmallVector<BasicBlock *> LoopLatchBlocks;
L->getLoopLatches(LoopLatchBlocks);
for (BasicBlock *ControlBB : LoopLatchBlocks) {
BranchInst *Br = dyn_cast<BranchInst>(ControlBB->getTerminator());
>From 075a778a6b5d7225c938c5bd2e7301e586f49318 Mon Sep 17 00:00:00 2001
From: Karthika Devi C <quic_kartc at quicinc.com>
Date: Wed, 30 Oct 2024 10:27:26 +0530
Subject: [PATCH 4/8] Apply suggestions from code review
Co-authored-by: Michael Kruse <github at meinersbur.de>
---
polly/include/polly/CodeGen/IRBuilder.h | 14 ++++++---
polly/lib/CodeGen/CodeGeneration.cpp | 31 ++++++++++---------
polly/lib/CodeGen/IRBuilder.cpp | 17 +++++-----
polly/lib/CodeGen/LoopGenerators.cpp | 19 ++++++------
.../CodeGen/Metadata/basic_vec_annotate.ll | 4 +--
5 files changed, 49 insertions(+), 36 deletions(-)
diff --git a/polly/include/polly/CodeGen/IRBuilder.h b/polly/include/polly/CodeGen/IRBuilder.h
index 73571dbcf1f2bd..047be8658c7922 100644
--- a/polly/include/polly/CodeGen/IRBuilder.h
+++ b/polly/include/polly/CodeGen/IRBuilder.h
@@ -58,10 +58,16 @@ class ScopAnnotator {
/// Annotate the new instruction @p I for all parallel loops.
void annotate(llvm::Instruction *I);
- /// Annotate the loop latch @p B wrt. @p L.
- void annotateLoopLatch(llvm::BranchInst *B, llvm::Loop *L, bool IsParallel,
- bool setVectorizeMetadata,
- bool EnableLoopVectorizer) const;
+ /// Add vectorize metadata to the list @p Args after setting it to
+ /// @p EnableLoopVectorizer
+ void addVectorizeMetadata(llvm::LLVMContext &Ctx,
+ llvm::SmallVector<llvm::Metadata *, 3> *Args,
+ bool EnableLoopVectorizer) const;
+
+ /// Annotate the loop latch @p B.
+ void annotateLoopLatch(
+ llvm::BranchInst *B, bool IsParallel,
+ std::optional<bool> EnableVectorizeMetadata = std::nullopt) const;
/// Add alternative alias based pointers
///
diff --git a/polly/lib/CodeGen/CodeGeneration.cpp b/polly/lib/CodeGen/CodeGeneration.cpp
index be9364da0fa08b..80a44c892f4d24 100644
--- a/polly/lib/CodeGen/CodeGeneration.cpp
+++ b/polly/lib/CodeGen/CodeGeneration.cpp
@@ -242,20 +242,23 @@ static bool generateCode(Scop &S, IslAstInfo &AI, LoopInfo &LI,
// for the code flow taken when RTCs fail. Because we don't want the
// Loop Vectorizer to come in later and vectorize the original fall back
// loop when 'polly-annotate-metadata-vectorize' is passed.
- if (PollyVectorizeMetadata && &Annotator) {
- for (Loop *L : LI.getLoopsInPreorder()) {
- if (S.contains(L)) {
- Annotator.pushLoop(L, false);
- SmallVector<BasicBlock *, 4> LoopLatchBlocks;
- L->getLoopLatches(LoopLatchBlocks);
- for (BasicBlock *ControlBB : LoopLatchBlocks) {
- BranchInst *Br = dyn_cast<BranchInst>(ControlBB->getTerminator());
- if (Br)
- Annotator.annotateLoopLatch(Br, L, false, true, false);
- }
- Annotator.popLoop(false);
- }
- }
+ if (PollyVectorizeMetadata) {
+ LLVMContext &Ctx = S.getFunction().getContext();
+
+ if (!L || !S.contains(L))
+ continue;
+ MDNode *LoopID = L->getLoopID();
+ SmallVector<Metadata *, 3> Args;
+ if (LoopID)
+ for (unsigned i = 0, e = LoopID->getNumOperands(); i != e; ++i)
+ Args.push_back(LoopID->getOperand(i));
+ else
+ Args.push_back(nullptr);
+
+ Annotator.addVectorizeMetadata(Ctx, &Args, false);
+ MDNode *NewLoopID = MDNode::get(Ctx, Args);
+ NewLoopID->replaceOperandWith(0, NewLoopID);
+ L->setLoopID(NewLoopID);
}
if (PerfMonitoring) {
diff --git a/polly/lib/CodeGen/IRBuilder.cpp b/polly/lib/CodeGen/IRBuilder.cpp
index 4e01b86da563a8..e915fc6d84f2d7 100644
--- a/polly/lib/CodeGen/IRBuilder.cpp
+++ b/polly/lib/CodeGen/IRBuilder.cpp
@@ -128,8 +128,9 @@ void ScopAnnotator::popLoop(bool IsParallel) {
LoopAttrEnv.pop_back();
}
-void addVectorizeMetadata(LLVMContext &Ctx, SmallVector<Metadata *, 3> *Args,
- bool EnableLoopVectorizer) {
+void ScopAnnotator::addVectorizeMetadata(LLVMContext &Ctx,
+ SmallVector<Metadata *, 3> *Args,
+ bool EnableLoopVectorizer) const {
MDString *PropName = MDString::get(Ctx, "llvm.loop.vectorize.enable");
ConstantInt *Value =
ConstantInt::get(Type::getInt1Ty(Ctx), EnableLoopVectorizer);
@@ -144,9 +145,11 @@ void addParallelMetadata(LLVMContext &Ctx, SmallVector<Metadata *, 3> *Args,
Args->push_back(MDNode::get(Ctx, {PropName, AccGroup}));
}
-void ScopAnnotator::annotateLoopLatch(BranchInst *B, Loop *L, bool IsParallel,
- bool setVectorizeMetadata,
- bool EnableLoopVectorizer) const {
+// Last argument is optional, if no value is passed, we don't annotate
+// any vectorize metadata.
+void ScopAnnotator::annotateLoopLatch(
+ BranchInst *B, bool IsParallel,
+ std::optional<bool> EnableVectorizeMetadata) const {
LLVMContext &Ctx = SE->getContext();
SmallVector<Metadata *, 3> Args;
@@ -164,8 +167,8 @@ void ScopAnnotator::annotateLoopLatch(BranchInst *B, Loop *L, bool IsParallel,
}
if (IsParallel)
addParallelMetadata(Ctx, &Args, ParallelLoops);
- if (setVectorizeMetadata)
- addVectorizeMetadata(Ctx, &Args, EnableLoopVectorizer);
+ if (EnableVectorizeMetadata.has_value())
+ this->addVectorizeMetadata(Ctx, &Args, *EnableVectorizeMetadata);
// No metadata to annotate.
if (!MData && Args.size() <= 1)
diff --git a/polly/lib/CodeGen/LoopGenerators.cpp b/polly/lib/CodeGen/LoopGenerators.cpp
index c082b61096250d..02491074093f23 100644
--- a/polly/lib/CodeGen/LoopGenerators.cpp
+++ b/polly/lib/CodeGen/LoopGenerators.cpp
@@ -163,17 +163,18 @@ Value *polly::createLoop(Value *LB, Value *UB, Value *Stride,
// Create the loop latch and annotate it as such.
BranchInst *B = Builder.CreateCondBr(LoopCondition, HeaderBB, ExitBB);
- // If the 'polly-annotate-metadata-vectorize' flag is passed, we add
- // the vectorize metadata. Otherwise we fall back to previous behavior
- // of annotating the loop only when LoopVectDisabled is true.
+ // Don't annotate vectorize metadata when both LoopVectDisabled and
+ // PollyVectorizeMetadata are disabled. Annotate vectorize metadata to false
+ // when LoopVectDisabled is true. Otherwise we annotate the vectorize metadata
+ // to true.
if (Annotator) {
- if (PollyVectorizeMetadata)
- Annotator->annotateLoopLatch(B, NewLoop, Parallel, true,
- !LoopVectDisabled);
- else if (LoopVectDisabled)
- Annotator->annotateLoopLatch(B, NewLoop, Parallel, true, false);
+ if (!LoopVectDisabled && !PollyVectorizeMetadata)
+ Annotator->annotateLoopLatch(B, Parallel);
else
- Annotator->annotateLoopLatch(B, NewLoop, Parallel, false, false);
+ Annotator->annotateLoopLatch(
+ B, Parallel,
+ /*EnableVectorizeMetadata*/ !LoopVectDisabled &&
+ PollyVectorizeMetadata);
}
IV->addIncoming(IncrementedIV, HeaderBB);
diff --git a/polly/test/CodeGen/Metadata/basic_vec_annotate.ll b/polly/test/CodeGen/Metadata/basic_vec_annotate.ll
index b4b1a5c8160264..5edb495e0faffe 100644
--- a/polly/test/CodeGen/Metadata/basic_vec_annotate.ll
+++ b/polly/test/CodeGen/Metadata/basic_vec_annotate.ll
@@ -12,8 +12,8 @@
; CHECK: br {{.*}} !llvm.loop [[LOOP:![0-9]+]]
; CHECK: polly.stmt.for.body:
; CHECK: br {{.*}} !llvm.loop [[POLLY_LOOP:![0-9]+]]
-; CHECK: [[LOOP]] = distinct !{[[LOOP]], [[META2:![0-9]+]]}
-; CHECK: [[META2]] = !{!"llvm.loop.vectorize.enable", i1 false}
+; CHECK: [[LOOP]] = distinct !{[[LOOP]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
+; CHECK: [[META3]] = !{!"llvm.loop.vectorize.enable", i1 false}
; CHECK: [[POLLY_LOOP]] = distinct !{[[POLLY_LOOP]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
; CHECK: [[META3]] = !{!"llvm.loop.vectorize.enable", i1 true}
>From c368310fdd94fcf15b5b5b727fa25f6726f89eff Mon Sep 17 00:00:00 2001
From: Karthika Devi C <quic_kartc at quicinc.com>
Date: Tue, 5 Nov 2024 16:47:13 +0530
Subject: [PATCH 5/8] Update CodeGeneration.cpp
---
polly/lib/CodeGen/CodeGeneration.cpp | 31 ++++++++++++++--------------
1 file changed, 16 insertions(+), 15 deletions(-)
diff --git a/polly/lib/CodeGen/CodeGeneration.cpp b/polly/lib/CodeGen/CodeGeneration.cpp
index 80a44c892f4d24..1cf13542fab407 100644
--- a/polly/lib/CodeGen/CodeGeneration.cpp
+++ b/polly/lib/CodeGen/CodeGeneration.cpp
@@ -244,21 +244,22 @@ static bool generateCode(Scop &S, IslAstInfo &AI, LoopInfo &LI,
// loop when 'polly-annotate-metadata-vectorize' is passed.
if (PollyVectorizeMetadata) {
LLVMContext &Ctx = S.getFunction().getContext();
-
- if (!L || !S.contains(L))
- continue;
- MDNode *LoopID = L->getLoopID();
- SmallVector<Metadata *, 3> Args;
- if (LoopID)
- for (unsigned i = 0, e = LoopID->getNumOperands(); i != e; ++i)
- Args.push_back(LoopID->getOperand(i));
- else
- Args.push_back(nullptr);
-
- Annotator.addVectorizeMetadata(Ctx, &Args, false);
- MDNode *NewLoopID = MDNode::get(Ctx, Args);
- NewLoopID->replaceOperandWith(0, NewLoopID);
- L->setLoopID(NewLoopID);
+ for (Loop *L : LI.getLoopsInPreorder()) {
+ if (!L || !S.contains(L))
+ continue;
+ MDNode *LoopID = L->getLoopID();
+ SmallVector<Metadata *, 3> Args;
+ if (LoopID)
+ for (unsigned i = 0, e = LoopID->getNumOperands(); i != e; ++i)
+ Args.push_back(LoopID->getOperand(i));
+ else
+ Args.push_back(nullptr);
+
+ Annotator.addVectorizeMetadata(Ctx, &Args, false);
+ MDNode *NewLoopID = MDNode::get(Ctx, Args);
+ NewLoopID->replaceOperandWith(0, NewLoopID);
+ L->setLoopID(NewLoopID);
+ }
}
if (PerfMonitoring) {
>From 92703e2a56f50e310ffe6b0d88710698ccc5b8cc Mon Sep 17 00:00:00 2001
From: Karthika Devi C <quic_kartc at quicinc.com>
Date: Wed, 18 Dec 2024 05:07:31 -0800
Subject: [PATCH 6/8] Remove changes that are already made in #119188
---
polly/include/polly/CodeGen/IRBuilder.h | 8 ++-----
polly/lib/CodeGen/CodeGeneration.cpp | 29 -------------------------
polly/lib/CodeGen/IRBuilder.cpp | 6 ++---
polly/lib/CodeGen/LoopGenerators.cpp | 19 ++++++++--------
4 files changed, 14 insertions(+), 48 deletions(-)
diff --git a/polly/include/polly/CodeGen/IRBuilder.h b/polly/include/polly/CodeGen/IRBuilder.h
index 047be8658c7922..6641ac9a0c0684 100644
--- a/polly/include/polly/CodeGen/IRBuilder.h
+++ b/polly/include/polly/CodeGen/IRBuilder.h
@@ -58,13 +58,9 @@ class ScopAnnotator {
/// Annotate the new instruction @p I for all parallel loops.
void annotate(llvm::Instruction *I);
- /// Add vectorize metadata to the list @p Args after setting it to
- /// @p EnableLoopVectorizer
- void addVectorizeMetadata(llvm::LLVMContext &Ctx,
- llvm::SmallVector<llvm::Metadata *, 3> *Args,
- bool EnableLoopVectorizer) const;
-
/// Annotate the loop latch @p B.
+ /// Last argument is optional, if no value is passed, we don't annotate
+ /// any vectorize metadata.
void annotateLoopLatch(
llvm::BranchInst *B, bool IsParallel,
std::optional<bool> EnableVectorizeMetadata = std::nullopt) const;
diff --git a/polly/lib/CodeGen/CodeGeneration.cpp b/polly/lib/CodeGen/CodeGeneration.cpp
index 1cf13542fab407..8813cfd959ef6e 100644
--- a/polly/lib/CodeGen/CodeGeneration.cpp
+++ b/polly/lib/CodeGen/CodeGeneration.cpp
@@ -54,11 +54,6 @@ static cl::opt<bool> Verify("polly-codegen-verify",
cl::desc("Verify the function generated by Polly"),
cl::Hidden, cl::cat(PollyCategory));
-cl::opt<bool> PollyVectorizeMetadata(
- "polly-annotate-metadata-vectorize",
- cl::desc("Append vectorize enable/disable metadata from polly"),
- cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
-
bool polly::PerfMonitoring;
static cl::opt<bool, true>
@@ -238,30 +233,6 @@ static bool generateCode(Scop &S, IslAstInfo &AI, LoopInfo &LI,
NodeBuilder.allocateNewArrays(StartExitBlocks);
Annotator.buildAliasScopes(S);
- // The code below annotates the "llvm.loop.vectorize.enable" to false
- // for the code flow taken when RTCs fail. Because we don't want the
- // Loop Vectorizer to come in later and vectorize the original fall back
- // loop when 'polly-annotate-metadata-vectorize' is passed.
- if (PollyVectorizeMetadata) {
- LLVMContext &Ctx = S.getFunction().getContext();
- for (Loop *L : LI.getLoopsInPreorder()) {
- if (!L || !S.contains(L))
- continue;
- MDNode *LoopID = L->getLoopID();
- SmallVector<Metadata *, 3> Args;
- if (LoopID)
- for (unsigned i = 0, e = LoopID->getNumOperands(); i != e; ++i)
- Args.push_back(LoopID->getOperand(i));
- else
- Args.push_back(nullptr);
-
- Annotator.addVectorizeMetadata(Ctx, &Args, false);
- MDNode *NewLoopID = MDNode::get(Ctx, Args);
- NewLoopID->replaceOperandWith(0, NewLoopID);
- L->setLoopID(NewLoopID);
- }
- }
-
if (PerfMonitoring) {
PerfMonitor P(S, EnteringBB->getParent()->getParent());
P.initialize();
diff --git a/polly/lib/CodeGen/IRBuilder.cpp b/polly/lib/CodeGen/IRBuilder.cpp
index e915fc6d84f2d7..bbd7e823733c14 100644
--- a/polly/lib/CodeGen/IRBuilder.cpp
+++ b/polly/lib/CodeGen/IRBuilder.cpp
@@ -128,9 +128,9 @@ void ScopAnnotator::popLoop(bool IsParallel) {
LoopAttrEnv.pop_back();
}
-void ScopAnnotator::addVectorizeMetadata(LLVMContext &Ctx,
+static void addVectorizeMetadata(LLVMContext &Ctx,
SmallVector<Metadata *, 3> *Args,
- bool EnableLoopVectorizer) const {
+ bool EnableLoopVectorizer) {
MDString *PropName = MDString::get(Ctx, "llvm.loop.vectorize.enable");
ConstantInt *Value =
ConstantInt::get(Type::getInt1Ty(Ctx), EnableLoopVectorizer);
@@ -145,8 +145,6 @@ void addParallelMetadata(LLVMContext &Ctx, SmallVector<Metadata *, 3> *Args,
Args->push_back(MDNode::get(Ctx, {PropName, AccGroup}));
}
-// Last argument is optional, if no value is passed, we don't annotate
-// any vectorize metadata.
void ScopAnnotator::annotateLoopLatch(
BranchInst *B, bool IsParallel,
std::optional<bool> EnableVectorizeMetadata) const {
diff --git a/polly/lib/CodeGen/LoopGenerators.cpp b/polly/lib/CodeGen/LoopGenerators.cpp
index 02491074093f23..5f772170d96282 100644
--- a/polly/lib/CodeGen/LoopGenerators.cpp
+++ b/polly/lib/CodeGen/LoopGenerators.cpp
@@ -14,7 +14,6 @@
#include "polly/CodeGen/LoopGenerators.h"
#include "polly/Options.h"
#include "polly/ScopDetection.h"
-#include "polly/ScopInfo.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfoMetadata.h"
@@ -36,7 +35,10 @@ static cl::opt<int, true>
cl::Hidden, cl::location(polly::PollyNumThreads),
cl::init(0), cl::cat(PollyCategory));
-extern cl::opt<bool> PollyVectorizeMetadata;
+cl::opt<bool> PollyVectorizeMetadata(
+ "polly-annotate-metadata-vectorize",
+ cl::desc("Append vectorize enable/disable metadata from polly"),
+ cl::init(false), cl::ZeroOrMore, cl::cat(PollyCategory));
static cl::opt<OMPGeneralSchedulingType, true> XPollyScheduling(
"polly-scheduling",
@@ -168,13 +170,12 @@ Value *polly::createLoop(Value *LB, Value *UB, Value *Stride,
// when LoopVectDisabled is true. Otherwise we annotate the vectorize metadata
// to true.
if (Annotator) {
- if (!LoopVectDisabled && !PollyVectorizeMetadata)
- Annotator->annotateLoopLatch(B, Parallel);
- else
- Annotator->annotateLoopLatch(
- B, Parallel,
- /*EnableVectorizeMetadata*/ !LoopVectDisabled &&
- PollyVectorizeMetadata);
+ std::optional<bool> EnableVectorizeMetadata;
+ if (LoopVectDisabled)
+ EnableVectorizeMetadata = false;
+ else if (PollyVectorizeMetadata)
+ EnableVectorizeMetadata = true;
+ Annotator->annotateLoopLatch(B, Parallel, EnableVectorizeMetadata);
}
IV->addIncoming(IncrementedIV, HeaderBB);
>From 7a51d22703e9ff3892e85ab61213ec48878f1fbb Mon Sep 17 00:00:00 2001
From: Karthika Devi C <quic_kartc at quicinc.com>
Date: Wed, 18 Dec 2024 22:41:11 +0530
Subject: [PATCH 7/8] Update IRBuilder.cpp
---
polly/lib/CodeGen/IRBuilder.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/polly/lib/CodeGen/IRBuilder.cpp b/polly/lib/CodeGen/IRBuilder.cpp
index bbd7e823733c14..65e458cc5db4d7 100644
--- a/polly/lib/CodeGen/IRBuilder.cpp
+++ b/polly/lib/CodeGen/IRBuilder.cpp
@@ -129,8 +129,8 @@ void ScopAnnotator::popLoop(bool IsParallel) {
}
static void addVectorizeMetadata(LLVMContext &Ctx,
- SmallVector<Metadata *, 3> *Args,
- bool EnableLoopVectorizer) {
+ SmallVector<Metadata *, 3> *Args,
+ bool EnableLoopVectorizer) {
MDString *PropName = MDString::get(Ctx, "llvm.loop.vectorize.enable");
ConstantInt *Value =
ConstantInt::get(Type::getInt1Ty(Ctx), EnableLoopVectorizer);
>From 1a07e39507202e14cd04e4ee377cc8a68542e9d8 Mon Sep 17 00:00:00 2001
From: Karthika Devi C <quic_kartc at quicinc.com>
Date: Wed, 18 Dec 2024 22:41:42 +0530
Subject: [PATCH 8/8] Update basic_vec_annotate.ll
---
polly/test/CodeGen/Metadata/basic_vec_annotate.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/polly/test/CodeGen/Metadata/basic_vec_annotate.ll b/polly/test/CodeGen/Metadata/basic_vec_annotate.ll
index 5edb495e0faffe..ebe91636ea3cc2 100644
--- a/polly/test/CodeGen/Metadata/basic_vec_annotate.ll
+++ b/polly/test/CodeGen/Metadata/basic_vec_annotate.ll
@@ -13,7 +13,7 @@
; CHECK: polly.stmt.for.body:
; CHECK: br {{.*}} !llvm.loop [[POLLY_LOOP:![0-9]+]]
; CHECK: [[LOOP]] = distinct !{[[LOOP]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
-; CHECK: [[META3]] = !{!"llvm.loop.vectorize.enable", i1 false}
+; CHECK: [[META3]] = !{!"llvm.loop.vectorize.enable", i32 0}
; CHECK: [[POLLY_LOOP]] = distinct !{[[POLLY_LOOP]], [[META2:![0-9]+]], [[META3:![0-9]+]]}
; CHECK: [[META3]] = !{!"llvm.loop.vectorize.enable", i1 true}
More information about the llvm-commits
mailing list