[llvm] [LoadStoreVectorizer] Postprocess and merge equivalence classes (PR #114501)
Vyacheslav Klochkov via llvm-commits
llvm-commits at lists.llvm.org
Wed Dec 11 17:46:44 PST 2024
https://github.com/v-klochkov updated https://github.com/llvm/llvm-project/pull/114501
>From 0a16db0dfd12997a769053ea2c47f9ebcdfb1ea9 Mon Sep 17 00:00:00 2001
From: "Klochkov, Vyacheslav N" <vyacheslav.n.klochkov at intel.com>
Date: Wed, 30 Oct 2024 19:29:41 -0700
Subject: [PATCH 1/5] [LoadStoreVectorizer] Postprocess and merge equivalence
classes
This patch introduces a new method:
void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const
The method is called at the end of Vectorizer::collectEquivalenceClasses() and is
needed to merge equivalence classes that differ only by their underlying objects
(UB1 and UB2), where UB1 is 1-level-indirection underlying base for UB2.
This situation arises due to the limited lookup depth used during the search of
underlying bases with llvm::getUnderlyingObject(ptr).
Using any fixed lookup depth can result into creation of multiple equivalence
classes that only differ by 1-level indirection bases.
The new approach merges equivalence classes if they have adjucent bases (1-level indirection).
If a series of equivalence classes form ladder formed of 1-step/level indirections,
they are all merged into a single equivalence class.
This provides more opportunities for the load-store vectorizer to generate better vectors.
Signed-off-by: Klochkov, Vyacheslav N <vyacheslav.n.klochkov at intel.com>
---
.../Vectorize/LoadStoreVectorizer.cpp | 128 ++++++++++++++++++
.../X86/massive_indirection.ll | 63 +++++++++
2 files changed, 191 insertions(+)
create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 02ec1d5c259cd6..59c7f2239d972a 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -324,6 +324,11 @@ class Vectorizer {
Instruction *ChainElem, Instruction *ChainBegin,
const DenseMap<Instruction *, APInt /*OffsetFromLeader*/> &ChainOffsets);
+ /// Merges the equivalence classes if they have uderlying objects that differ
+ /// by one level of indirection (i.e., one is a getelementptr and the other is
+ /// the base pointer in that getelementptr).
+ void mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const;
+
/// Collects loads and stores grouped by "equivalence class", where:
/// - all elements in an eq class are a load or all are a store,
/// - they all load/store the same element size (it's OK to have e.g. i8 and
@@ -1305,6 +1310,128 @@ std::optional<APInt> Vectorizer::getConstantOffsetSelects(
return std::nullopt;
}
+void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
+ if (EQClasses.size() < 2) // There is nothing to merge.
+ return;
+
+ // The reduced key has all elements of the ECClassKey except the underlying
+ // object. Check that EqClassKey has 4 elements and define the reduced key.
+ static_assert(std::tuple_size_v<EqClassKey> == 4,
+ "EqClassKey has changed - EqClassReducedKey needs changes too");
+ using EqClassReducedKey =
+ std::tuple<std::tuple_element_t<1, EqClassKey> /* AddrSpace */,
+ std::tuple_element_t<2, EqClassKey> /* Element size */,
+ std::tuple_element_t<3, EqClassKey> /* IsLoad; */>;
+ using ECReducedKeyToUnderlyingObjectMap =
+ MapVector<EqClassReducedKey,
+ SmallPtrSet<std::tuple_element_t<0, EqClassKey>, 4>>;
+
+ // Form a map from the reduced key (without the underlying object) to the
+ // underlying objects: 1 reduced key to many underlying objects, to form
+ // groups of potentially merge-able equivalence classes.
+ ECReducedKeyToUnderlyingObjectMap RedKeyToUOMap;
+ bool FoundPotentiallyOptimizableEC = false;
+ for (const auto &EC : EQClasses) {
+ const auto &Key = EC.first;
+ EqClassReducedKey RedKey{std::get<1>(Key), std::get<2>(Key),
+ std::get<3>(Key)};
+ RedKeyToUOMap[RedKey].insert(std::get<0>(Key));
+ if (RedKeyToUOMap[RedKey].size() > 1)
+ FoundPotentiallyOptimizableEC = true;
+ }
+ if (!FoundPotentiallyOptimizableEC)
+ return;
+
+ LLVM_DEBUG({
+ dbgs() << "LSV: mergeEquivalenceClasses: before merging:\n";
+ for (const auto &EC : EQClasses) {
+ dbgs() << " Key: ([" << std::get<0>(EC.first)
+ << "]: " << *std::get<0>(EC.first) << ", " << std::get<1>(EC.first)
+ << ", " << std::get<2>(EC.first) << ", "
+ << static_cast<int>(std::get<3>(EC.first)) << ")\n";
+ for (const auto &Inst : EC.second)
+ dbgs() << "\tInst:\t" << *Inst << "\n";
+ }
+ });
+ LLVM_DEBUG({
+ dbgs() << "LSV: mergeEquivalenceClasses: RedKeyToUOMap:\n";
+ for (const auto &RedKeyToUO : RedKeyToUOMap) {
+ dbgs() << " Reduced key: (" << std::get<0>(RedKeyToUO.first) << ", "
+ << std::get<1>(RedKeyToUO.first) << ", "
+ << static_cast<int>(std::get<2>(RedKeyToUO.first)) << ") --> "
+ << RedKeyToUO.second.size() << " underlying objects:\n";
+ for (auto UObject : RedKeyToUO.second)
+ dbgs() << " [" << UObject << "]: " << *UObject << "\n";
+ }
+ });
+
+ using UObjectToUObjectMap = DenseMap<const Value *, const Value *>;
+
+ // Compute the ultimate targets for a set of underlying objects.
+ auto GetUltimateTargets =
+ [](SmallPtrSetImpl<const Value *> &UObjects) -> UObjectToUObjectMap {
+ UObjectToUObjectMap IndirectionMap;
+ for (const auto *UObject : UObjects) {
+ const unsigned MaxLookupDepth = 1; // look for 1-level indirections only
+ const auto *UltimateTarget =
+ llvm::getUnderlyingObject(UObject, MaxLookupDepth);
+ if (UltimateTarget != UObject)
+ IndirectionMap[UObject] = UltimateTarget;
+ }
+ UObjectToUObjectMap UltimateTargetsMap;
+ for (const auto *UObject : UObjects) {
+ auto Target = UObject;
+ auto It = IndirectionMap.find(Target);
+ for (; It != IndirectionMap.end(); It = IndirectionMap.find(Target))
+ Target = It->second;
+ UltimateTargetsMap[UObject] = Target;
+ }
+ return UltimateTargetsMap;
+ };
+
+ // For each item in RedKeyToUOMap, if it has more than one underlying object,
+ // try to merge the equivalence classes.
+ for (auto &RedKeyToUO : RedKeyToUOMap) {
+ auto UObjects = RedKeyToUO.second;
+ if (UObjects.size() < 2)
+ continue;
+ const auto RedKey = RedKeyToUO.first;
+ auto UTMap = GetUltimateTargets(UObjects);
+ for (const auto &UT : UTMap) {
+ const Value *UObject = UT.first;
+ const Value *UltimateTarget = UT.second;
+ if (UObject == UltimateTarget)
+ continue;
+
+ EqClassKey KeyFrom{UObject, std::get<0>(RedKey), std::get<1>(RedKey),
+ std::get<2>(RedKey)};
+ EqClassKey KeyTo{UltimateTarget, std::get<0>(RedKey), std::get<1>(RedKey),
+ std::get<2>(RedKey)};
+ auto VecFrom = EQClasses[KeyFrom];
+ auto VecTo = EQClasses[KeyTo];
+ SmallVector<Instruction *, 8> MergedVec;
+ std::merge(VecFrom.begin(), VecFrom.end(), VecTo.begin(), VecTo.end(),
+ std::back_inserter(MergedVec),
+ [](Instruction *A, Instruction *B) {
+ return A && B && A->comesBefore(B);
+ });
+ EQClasses[KeyTo] = std::move(MergedVec);
+ EQClasses.erase(KeyFrom);
+ }
+ }
+ LLVM_DEBUG({
+ dbgs() << "LSV: mergeEquivalenceClasses: after merging:\n";
+ for (const auto &EC : EQClasses) {
+ dbgs() << " Key: ([" << std::get<0>(EC.first)
+ << "]: " << *std::get<0>(EC.first) << ", " << std::get<1>(EC.first)
+ << ", " << std::get<2>(EC.first) << ", "
+ << static_cast<int>(std::get<3>(EC.first)) << ")\n";
+ for (const auto &Inst : EC.second)
+ dbgs() << "\tInst:\t" << *Inst << "\n";
+ }
+ });
+}
+
EquivalenceClassMap
Vectorizer::collectEquivalenceClasses(BasicBlock::iterator Begin,
BasicBlock::iterator End) {
@@ -1377,6 +1504,7 @@ Vectorizer::collectEquivalenceClasses(BasicBlock::iterator Begin,
.emplace_back(&I);
}
+ mergeEquivalenceClasses(Ret);
return Ret;
}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
new file mode 100644
index 00000000000000..ab320f02ed937d
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
@@ -0,0 +1,63 @@
+; RUN: opt %s -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -mcpu=skx -S -o %t.out.ll
+; RUN: FileCheck -input-file=%t.out.ll %s
+
+; This test verifies that the vectorizer can handle an extended sequence of
+; getelementptr instructions and generate longer vectors. With special handling,
+; some elements can still be vectorized even if they require looking up the
+; common underlying object deeper than 6 levels from the original pointer.
+
+; The test below is the simplified version of actual performance oriented
+; workload; the offsets in getelementptr instructins are similar or same for
+; the test simplicity.
+
+define void @v1_v2_v4_v1_to_v8_levels_6_7_8_8(i32 %arg0, ptr align 16 %arg1) {
+; CHECK-LABEL: @v1_v2_v4_v1_to_v8_levels_6_7_8_8
+; CHECK: store <8 x half>
+
+ %level1 = getelementptr inbounds i8, ptr %arg1, i32 917504
+ %level2 = getelementptr i8, ptr %level1, i32 %arg0
+ %level3 = getelementptr i8, ptr %level2, i32 32768
+ %level4 = getelementptr inbounds i8, ptr %level3, i32 %arg0
+ %level5 = getelementptr i8, ptr %level4, i32 %arg0
+
+ %a6 = getelementptr i8, ptr %level5, i32 %arg0
+ %b7 = getelementptr i8, ptr %a6, i32 2
+ %c8 = getelementptr i8, ptr %b7, i32 8
+ %d8 = getelementptr inbounds i8, ptr %b7, i32 12
+
+ store half 0xH0000, ptr %a6, align 16
+ store <4 x half> zeroinitializer, ptr %b7, align 2
+ store <2 x half> zeroinitializer, ptr %c8, align 2
+ store half 0xH0000, ptr %d8, align 2
+ ret void
+}
+
+define void @v1x8_levels_6_7_8_9_10_11_12_13(i32 %arg0, ptr align 16 %arg1) {
+; CHECK-LABEL: @v1x8_levels_6_7_8_9_10_11_12_13
+; CHECK: store <8 x half>
+
+ %level1 = getelementptr inbounds i8, ptr %arg1, i32 917504
+ %level2 = getelementptr i8, ptr %level1, i32 %arg0
+ %level3 = getelementptr i8, ptr %level2, i32 32768
+ %level4 = getelementptr inbounds i8, ptr %level3, i32 %arg0
+ %level5 = getelementptr i8, ptr %level4, i32 %arg0
+
+ %a6 = getelementptr i8, ptr %level5, i32 %arg0
+ %b7 = getelementptr i8, ptr %a6, i32 2
+ %c8 = getelementptr i8, ptr %b7, i32 2
+ %d9 = getelementptr inbounds i8, ptr %c8, i32 2
+ %e10 = getelementptr inbounds i8, ptr %d9, i32 2
+ %f11 = getelementptr inbounds i8, ptr %e10, i32 2
+ %g12 = getelementptr inbounds i8, ptr %f11, i32 2
+ %h13 = getelementptr inbounds i8, ptr %g12, i32 2
+
+ store half 0xH0000, ptr %a6, align 16
+ store half 0xH0000, ptr %b7, align 2
+ store half 0xH0000, ptr %c8, align 2
+ store half 0xH0000, ptr %d9, align 2
+ store half 0xH0000, ptr %e10, align 8
+ store half 0xH0000, ptr %f11, align 2
+ store half 0xH0000, ptr %g12, align 2
+ store half 0xH0000, ptr %h13, align 2
+ ret void
+}
>From 791a265c81655df7aabf0c31aa13be3d7d251beb Mon Sep 17 00:00:00 2001
From: "Klochkov, Vyacheslav N" <vyacheslav.n.klochkov at intel.com>
Date: Fri, 1 Nov 2024 09:09:04 -0700
Subject: [PATCH 2/5] [NFC] Address review comments: fix misprint + use C++17
struct binding
---
llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp | 8 ++------
.../LoadStoreVectorizer/X86/massive_indirection.ll | 2 +-
2 files changed, 3 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 59c7f2239d972a..699fea8872dc58 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1391,15 +1391,11 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
// For each item in RedKeyToUOMap, if it has more than one underlying object,
// try to merge the equivalence classes.
- for (auto &RedKeyToUO : RedKeyToUOMap) {
- auto UObjects = RedKeyToUO.second;
+ for (auto &[RedKey, UObjects] : RedKeyToUOMap) {
if (UObjects.size() < 2)
continue;
- const auto RedKey = RedKeyToUO.first;
auto UTMap = GetUltimateTargets(UObjects);
- for (const auto &UT : UTMap) {
- const Value *UObject = UT.first;
- const Value *UltimateTarget = UT.second;
+ for (const auto &[UObject, UltimateTarget] : UTMap) {
if (UObject == UltimateTarget)
continue;
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
index ab320f02ed937d..b909f354393fab 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
@@ -7,7 +7,7 @@
; common underlying object deeper than 6 levels from the original pointer.
; The test below is the simplified version of actual performance oriented
-; workload; the offsets in getelementptr instructins are similar or same for
+; workload; the offsets in getelementptr instructions are similar or same for
; the test simplicity.
define void @v1_v2_v4_v1_to_v8_levels_6_7_8_8(i32 %arg0, ptr align 16 %arg1) {
>From 2325b526040f99093f24558935e17496c20a2f2b Mon Sep 17 00:00:00 2001
From: "Klochkov, Vyacheslav N" <vyacheslav.n.klochkov at intel.com>
Date: Fri, 8 Nov 2024 12:45:12 -0800
Subject: [PATCH 3/5] [NFC] Minor fixes to address the 2nd round of review
comments
---
.../Vectorize/LoadStoreVectorizer.cpp | 15 +++---
.../X86/massive_indirection.ll | 50 +++++++++++++------
2 files changed, 41 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 699fea8872dc58..9f24181d5d1f6d 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -324,7 +324,7 @@ class Vectorizer {
Instruction *ChainElem, Instruction *ChainBegin,
const DenseMap<Instruction *, APInt /*OffsetFromLeader*/> &ChainOffsets);
- /// Merges the equivalence classes if they have uderlying objects that differ
+ /// Merges the equivalence classes if they have underlying objects that differ
/// by one level of indirection (i.e., one is a getelementptr and the other is
/// the base pointer in that getelementptr).
void mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const;
@@ -1350,7 +1350,7 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
<< ", " << std::get<2>(EC.first) << ", "
<< static_cast<int>(std::get<3>(EC.first)) << ")\n";
for (const auto &Inst : EC.second)
- dbgs() << "\tInst:\t" << *Inst << "\n";
+ dbgs() << "\tInst: " << *Inst << '\n';
}
});
LLVM_DEBUG({
@@ -1361,7 +1361,7 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
<< static_cast<int>(std::get<2>(RedKeyToUO.first)) << ") --> "
<< RedKeyToUO.second.size() << " underlying objects:\n";
for (auto UObject : RedKeyToUO.second)
- dbgs() << " [" << UObject << "]: " << *UObject << "\n";
+ dbgs() << " [" << UObject << "]: " << *UObject << '\n';
}
});
@@ -1373,8 +1373,7 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
UObjectToUObjectMap IndirectionMap;
for (const auto *UObject : UObjects) {
const unsigned MaxLookupDepth = 1; // look for 1-level indirections only
- const auto *UltimateTarget =
- llvm::getUnderlyingObject(UObject, MaxLookupDepth);
+ const auto *UltimateTarget = getUnderlyingObject(UObject, MaxLookupDepth);
if (UltimateTarget != UObject)
IndirectionMap[UObject] = UltimateTarget;
}
@@ -1403,8 +1402,8 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
std::get<2>(RedKey)};
EqClassKey KeyTo{UltimateTarget, std::get<0>(RedKey), std::get<1>(RedKey),
std::get<2>(RedKey)};
- auto VecFrom = EQClasses[KeyFrom];
- auto VecTo = EQClasses[KeyTo];
+ const auto &VecFrom = EQClasses[KeyFrom];
+ const auto &VecTo = EQClasses[KeyTo];
SmallVector<Instruction *, 8> MergedVec;
std::merge(VecFrom.begin(), VecFrom.end(), VecTo.begin(), VecTo.end(),
std::back_inserter(MergedVec),
@@ -1423,7 +1422,7 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
<< ", " << std::get<2>(EC.first) << ", "
<< static_cast<int>(std::get<3>(EC.first)) << ")\n";
for (const auto &Inst : EC.second)
- dbgs() << "\tInst:\t" << *Inst << "\n";
+ dbgs() << "\tInst: " << *Inst << '\n';
}
});
}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
index b909f354393fab..ff3f2c9e2ed8a4 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
@@ -1,5 +1,5 @@
-; RUN: opt %s -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -mcpu=skx -S -o %t.out.ll
-; RUN: FileCheck -input-file=%t.out.ll %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt %s -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -mcpu=skx -S -o - | FileCheck %s
; This test verifies that the vectorizer can handle an extended sequence of
; getelementptr instructions and generate longer vectors. With special handling,
@@ -11,19 +11,28 @@
; the test simplicity.
define void @v1_v2_v4_v1_to_v8_levels_6_7_8_8(i32 %arg0, ptr align 16 %arg1) {
-; CHECK-LABEL: @v1_v2_v4_v1_to_v8_levels_6_7_8_8
-; CHECK: store <8 x half>
+; CHECK-LABEL: define void @v1_v2_v4_v1_to_v8_levels_6_7_8_8(
+; CHECK-SAME: i32 [[ARG0:%.*]], ptr align 16 [[ARG1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[LEVEL1:%.*]] = getelementptr i8, ptr [[ARG1]], i32 917504
+; CHECK-NEXT: [[LEVEL2:%.*]] = getelementptr i8, ptr [[LEVEL1]], i32 [[ARG0]]
+; CHECK-NEXT: [[LEVEL3:%.*]] = getelementptr i8, ptr [[LEVEL2]], i32 32768
+; CHECK-NEXT: [[LEVEL4:%.*]] = getelementptr i8, ptr [[LEVEL3]], i32 [[ARG0]]
+; CHECK-NEXT: [[LEVEL5:%.*]] = getelementptr i8, ptr [[LEVEL4]], i32 [[ARG0]]
+; CHECK-NEXT: [[A6:%.*]] = getelementptr i8, ptr [[LEVEL5]], i32 [[ARG0]]
+; CHECK-NEXT: store <8 x half> zeroinitializer, ptr [[A6]], align 16
+; CHECK-NEXT: ret void
+;
- %level1 = getelementptr inbounds i8, ptr %arg1, i32 917504
+ %level1 = getelementptr i8, ptr %arg1, i32 917504
%level2 = getelementptr i8, ptr %level1, i32 %arg0
%level3 = getelementptr i8, ptr %level2, i32 32768
- %level4 = getelementptr inbounds i8, ptr %level3, i32 %arg0
+ %level4 = getelementptr i8, ptr %level3, i32 %arg0
%level5 = getelementptr i8, ptr %level4, i32 %arg0
%a6 = getelementptr i8, ptr %level5, i32 %arg0
%b7 = getelementptr i8, ptr %a6, i32 2
%c8 = getelementptr i8, ptr %b7, i32 8
- %d8 = getelementptr inbounds i8, ptr %b7, i32 12
+ %d8 = getelementptr i8, ptr %b7, i32 12
store half 0xH0000, ptr %a6, align 16
store <4 x half> zeroinitializer, ptr %b7, align 2
@@ -33,23 +42,32 @@ define void @v1_v2_v4_v1_to_v8_levels_6_7_8_8(i32 %arg0, ptr align 16 %arg1) {
}
define void @v1x8_levels_6_7_8_9_10_11_12_13(i32 %arg0, ptr align 16 %arg1) {
-; CHECK-LABEL: @v1x8_levels_6_7_8_9_10_11_12_13
-; CHECK: store <8 x half>
+; CHECK-LABEL: define void @v1x8_levels_6_7_8_9_10_11_12_13(
+; CHECK-SAME: i32 [[ARG0:%.*]], ptr align 16 [[ARG1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[LEVEL1:%.*]] = getelementptr i8, ptr [[ARG1]], i32 917504
+; CHECK-NEXT: [[LEVEL2:%.*]] = getelementptr i8, ptr [[LEVEL1]], i32 [[ARG0]]
+; CHECK-NEXT: [[LEVEL3:%.*]] = getelementptr i8, ptr [[LEVEL2]], i32 32768
+; CHECK-NEXT: [[LEVEL4:%.*]] = getelementptr i8, ptr [[LEVEL3]], i32 [[ARG0]]
+; CHECK-NEXT: [[LEVEL5:%.*]] = getelementptr i8, ptr [[LEVEL4]], i32 [[ARG0]]
+; CHECK-NEXT: [[A6:%.*]] = getelementptr i8, ptr [[LEVEL5]], i32 [[ARG0]]
+; CHECK-NEXT: store <8 x half> zeroinitializer, ptr [[A6]], align 16
+; CHECK-NEXT: ret void
+;
- %level1 = getelementptr inbounds i8, ptr %arg1, i32 917504
+ %level1 = getelementptr i8, ptr %arg1, i32 917504
%level2 = getelementptr i8, ptr %level1, i32 %arg0
%level3 = getelementptr i8, ptr %level2, i32 32768
- %level4 = getelementptr inbounds i8, ptr %level3, i32 %arg0
+ %level4 = getelementptr i8, ptr %level3, i32 %arg0
%level5 = getelementptr i8, ptr %level4, i32 %arg0
%a6 = getelementptr i8, ptr %level5, i32 %arg0
%b7 = getelementptr i8, ptr %a6, i32 2
%c8 = getelementptr i8, ptr %b7, i32 2
- %d9 = getelementptr inbounds i8, ptr %c8, i32 2
- %e10 = getelementptr inbounds i8, ptr %d9, i32 2
- %f11 = getelementptr inbounds i8, ptr %e10, i32 2
- %g12 = getelementptr inbounds i8, ptr %f11, i32 2
- %h13 = getelementptr inbounds i8, ptr %g12, i32 2
+ %d9 = getelementptr i8, ptr %c8, i32 2
+ %e10 = getelementptr i8, ptr %d9, i32 2
+ %f11 = getelementptr i8, ptr %e10, i32 2
+ %g12 = getelementptr i8, ptr %f11, i32 2
+ %h13 = getelementptr i8, ptr %g12, i32 2
store half 0xH0000, ptr %a6, align 16
store half 0xH0000, ptr %b7, align 2
>From 3cc04a284512fb51271e4d8deb7a1e83735a879b Mon Sep 17 00:00:00 2001
From: "Klochkov, Vyacheslav N" <vyacheslav.n.klochkov at intel.com>
Date: Fri, 22 Nov 2024 17:31:46 -0800
Subject: [PATCH 4/5] Add one more LIT test case
It is built from the original workload by copying the STORE instructions
to be optimized, and manually bulding missing IRs around them minimally
reproducing the context of the original STOREs in the full test.
---
.../X86/massive_indirection.ll | 66 +++++++++++++++++++
1 file changed, 66 insertions(+)
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
index ff3f2c9e2ed8a4..649dde379a0883 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
@@ -79,3 +79,69 @@ define void @v1x8_levels_6_7_8_9_10_11_12_13(i32 %arg0, ptr align 16 %arg1) {
store half 0xH0000, ptr %h13, align 2
ret void
}
+
+define void @v1_4_4_4_2_1_to_v8_8_levels_6_7(i32 %arg0, ptr addrspace(3) align 16 %arg1_ptr, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, half %arg6_half, half %arg7_half) {
+; CHECK-LABEL: define void @v1_4_4_4_2_1_to_v8_8_levels_6_7(
+; CHECK-SAME: i32 [[ARG0:%.*]], ptr addrspace(3) align 16 [[ARG1_PTR:%.*]], i32 [[ARG2:%.*]], i32 [[ARG3:%.*]], i32 [[ARG4:%.*]], i32 [[ARG5:%.*]], half [[ARG6_HALF:%.*]], half [[ARG7_HALF:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[ARG1_PTR]], i32 458752
+; CHECK-NEXT: br [[DOTPREHEADER11_PREHEADER:label %.*]]
+; CHECK: [[_PREHEADER11_PREHEADER:.*:]]
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i32 [[ARG0]], 6
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[TMP1]], i32 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[TMP3]], i32 [[ARG2]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[TMP4]], i32 [[ARG3]]
+; CHECK-NEXT: [[VEC2_INIT:%.*]] = insertelement <2 x half> undef, half [[ARG7_HALF]], i32 0
+; CHECK-NEXT: [[VEC2:%.*]] = shufflevector <2 x half> [[VEC2_INIT]], <2 x half> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[ARG0]], 2
+; CHECK-NEXT: br i1 [[CMP]], [[DOTLR_PH:label %.*]], [[DOTEXIT_POINT:label %.*]]
+; CHECK: [[_LR_PH:.*:]]
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[TMP5]], i32 [[ARG4]]
+; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[GEP]], i32 [[ARG5]]
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x half> poison, half [[ARG6_HALF]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x half> [[TMP7]], half 0xH0000, i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x half> [[TMP8]], half 0xH0000, i32 2
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x half> [[TMP9]], half 0xH0000, i32 3
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x half> [[TMP10]], half 0xH0000, i32 4
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x half> [[VEC2]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x half> [[TMP11]], half [[TMP12]], i32 5
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x half> [[VEC2]], i32 1
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x half> [[TMP13]], half [[TMP14]], i32 6
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x half> [[TMP15]], half [[ARG7_HALF]], i32 7
+; CHECK-NEXT: store <8 x half> [[TMP16]], ptr addrspace(3) [[TMP6]], align 2
+; CHECK-NEXT: br [[DOTEXIT_POINT]]
+; CHECK: [[_EXIT_POINT:.*:]]
+; CHECK-NEXT: ret void
+;
+ %37 = getelementptr inbounds i8, ptr addrspace(3) %arg1_ptr, i32 458752
+ br label %.preheader11.preheader
+
+.preheader11.preheader:
+ %258 = shl nuw nsw i32 %arg0, 6
+ %259 = getelementptr inbounds i8, ptr addrspace(3) %37, i32 %258
+
+ %268 = getelementptr inbounds i8, ptr addrspace(3) %259, i32 %arg2
+ %269 = getelementptr inbounds i8, ptr addrspace(3) %268, i32 %arg3
+
+ %vec2_init = insertelement <2 x half> undef, half %arg7_half, i32 0
+ %vec2 = shufflevector <2 x half> %vec2_init, <2 x half> undef, <2 x i32> zeroinitializer
+
+ %cmp = icmp sgt i32 %arg0, 2
+ br i1 %cmp, label %.lr.ph, label %.exit_point
+
+.lr.ph:
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %269, i32 %arg4
+
+ %1000 = getelementptr inbounds i8, ptr addrspace(3) %gep, i32 %arg5
+ %1002 = getelementptr inbounds i8, ptr addrspace(3) %1000, i32 2
+ %1010 = getelementptr inbounds i8, ptr addrspace(3) %1000, i32 10
+ %1014 = getelementptr inbounds i8, ptr addrspace(3) %1000, i32 14
+
+ store half %arg6_half, ptr addrspace(3) %1000, align 2
+ store <4 x half> zeroinitializer, ptr addrspace(3) %1002, align 2
+ store <2 x half> %vec2, ptr addrspace(3) %1010, align 2
+ store half %arg7_half, ptr addrspace(3) %1014, align 2
+ br label %.exit_point
+
+.exit_point:
+ ret void
+}
>From 4bff74970fc7d80dd5d261770d026c58b4b9a22f Mon Sep 17 00:00:00 2001
From: "Klochkov, Vyacheslav N" <vyacheslav.n.klochkov at intel.com>
Date: Wed, 11 Dec 2024 17:20:20 -0800
Subject: [PATCH 5/5] Fix LIT: Rename unnamed/numbered vregs in LIT; remove
usage of undef
---
.../X86/massive_indirection.ll | 41 ++++++++-----------
1 file changed, 18 insertions(+), 23 deletions(-)
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
index 649dde379a0883..c4b0d2e311d9d7 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
@@ -80,9 +80,9 @@ define void @v1x8_levels_6_7_8_9_10_11_12_13(i32 %arg0, ptr align 16 %arg1) {
ret void
}
-define void @v1_4_4_4_2_1_to_v8_8_levels_6_7(i32 %arg0, ptr addrspace(3) align 16 %arg1_ptr, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, half %arg6_half, half %arg7_half) {
+define void @v1_4_4_4_2_1_to_v8_8_levels_6_7(i32 %arg0, ptr addrspace(3) align 16 %arg1_ptr, i32 %arg2, i32 %arg3, i32 %arg4, i32 %arg5, half %arg6_half, half %arg7_half, <2 x half> %arg8_2xhalf) {
; CHECK-LABEL: define void @v1_4_4_4_2_1_to_v8_8_levels_6_7(
-; CHECK-SAME: i32 [[ARG0:%.*]], ptr addrspace(3) align 16 [[ARG1_PTR:%.*]], i32 [[ARG2:%.*]], i32 [[ARG3:%.*]], i32 [[ARG4:%.*]], i32 [[ARG5:%.*]], half [[ARG6_HALF:%.*]], half [[ARG7_HALF:%.*]]) #[[ATTR0]] {
+; CHECK-SAME: i32 [[ARG0:%.*]], ptr addrspace(3) align 16 [[ARG1_PTR:%.*]], i32 [[ARG2:%.*]], i32 [[ARG3:%.*]], i32 [[ARG4:%.*]], i32 [[ARG5:%.*]], half [[ARG6_HALF:%.*]], half [[ARG7_HALF:%.*]], <2 x half> [[ARG8_2XHALF:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[ARG1_PTR]], i32 458752
; CHECK-NEXT: br [[DOTPREHEADER11_PREHEADER:label %.*]]
; CHECK: [[_PREHEADER11_PREHEADER:.*:]]
@@ -90,8 +90,6 @@ define void @v1_4_4_4_2_1_to_v8_8_levels_6_7(i32 %arg0, ptr addrspace(3) align 1
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[TMP1]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[TMP3]], i32 [[ARG2]]
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[TMP4]], i32 [[ARG3]]
-; CHECK-NEXT: [[VEC2_INIT:%.*]] = insertelement <2 x half> undef, half [[ARG7_HALF]], i32 0
-; CHECK-NEXT: [[VEC2:%.*]] = shufflevector <2 x half> [[VEC2_INIT]], <2 x half> undef, <2 x i32> zeroinitializer
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[ARG0]], 2
; CHECK-NEXT: br i1 [[CMP]], [[DOTLR_PH:label %.*]], [[DOTEXIT_POINT:label %.*]]
; CHECK: [[_LR_PH:.*:]]
@@ -102,9 +100,9 @@ define void @v1_4_4_4_2_1_to_v8_8_levels_6_7(i32 %arg0, ptr addrspace(3) align 1
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x half> [[TMP8]], half 0xH0000, i32 2
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x half> [[TMP9]], half 0xH0000, i32 3
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x half> [[TMP10]], half 0xH0000, i32 4
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x half> [[VEC2]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x half> [[ARG8_2XHALF]], i32 0
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x half> [[TMP11]], half [[TMP12]], i32 5
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x half> [[VEC2]], i32 1
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x half> [[ARG8_2XHALF]], i32 1
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x half> [[TMP13]], half [[TMP14]], i32 6
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x half> [[TMP15]], half [[ARG7_HALF]], i32 7
; CHECK-NEXT: store <8 x half> [[TMP16]], ptr addrspace(3) [[TMP6]], align 2
@@ -112,34 +110,31 @@ define void @v1_4_4_4_2_1_to_v8_8_levels_6_7(i32 %arg0, ptr addrspace(3) align 1
; CHECK: [[_EXIT_POINT:.*:]]
; CHECK-NEXT: ret void
;
- %37 = getelementptr inbounds i8, ptr addrspace(3) %arg1_ptr, i32 458752
+ %base1 = getelementptr inbounds i8, ptr addrspace(3) %arg1_ptr, i32 458752
br label %.preheader11.preheader
.preheader11.preheader:
- %258 = shl nuw nsw i32 %arg0, 6
- %259 = getelementptr inbounds i8, ptr addrspace(3) %37, i32 %258
+ %base2 = shl nuw nsw i32 %arg0, 6
+ %base3 = getelementptr inbounds i8, ptr addrspace(3) %base1, i32 %base2
- %268 = getelementptr inbounds i8, ptr addrspace(3) %259, i32 %arg2
- %269 = getelementptr inbounds i8, ptr addrspace(3) %268, i32 %arg3
-
- %vec2_init = insertelement <2 x half> undef, half %arg7_half, i32 0
- %vec2 = shufflevector <2 x half> %vec2_init, <2 x half> undef, <2 x i32> zeroinitializer
+ %base4 = getelementptr inbounds i8, ptr addrspace(3) %base3, i32 %arg2
+ %base5 = getelementptr inbounds i8, ptr addrspace(3) %base4, i32 %arg3
%cmp = icmp sgt i32 %arg0, 2
br i1 %cmp, label %.lr.ph, label %.exit_point
.lr.ph:
- %gep = getelementptr inbounds i8, ptr addrspace(3) %269, i32 %arg4
+ %gep = getelementptr inbounds i8, ptr addrspace(3) %base5, i32 %arg4
- %1000 = getelementptr inbounds i8, ptr addrspace(3) %gep, i32 %arg5
- %1002 = getelementptr inbounds i8, ptr addrspace(3) %1000, i32 2
- %1010 = getelementptr inbounds i8, ptr addrspace(3) %1000, i32 10
- %1014 = getelementptr inbounds i8, ptr addrspace(3) %1000, i32 14
+ %dst = getelementptr inbounds i8, ptr addrspace(3) %gep, i32 %arg5
+ %dst_off2 = getelementptr inbounds i8, ptr addrspace(3) %dst, i32 2
+ %dst_off10 = getelementptr inbounds i8, ptr addrspace(3) %dst, i32 10
+ %dst_off14 = getelementptr inbounds i8, ptr addrspace(3) %dst, i32 14
- store half %arg6_half, ptr addrspace(3) %1000, align 2
- store <4 x half> zeroinitializer, ptr addrspace(3) %1002, align 2
- store <2 x half> %vec2, ptr addrspace(3) %1010, align 2
- store half %arg7_half, ptr addrspace(3) %1014, align 2
+ store half %arg6_half, ptr addrspace(3) %dst, align 2
+ store <4 x half> zeroinitializer, ptr addrspace(3) %dst_off2, align 2
+ store <2 x half> %arg8_2xhalf, ptr addrspace(3) %dst_off10, align 2
+ store half %arg7_half, ptr addrspace(3) %dst_off14, align 2
br label %.exit_point
.exit_point:
More information about the llvm-commits
mailing list