[llvm] faa631f - [LSV] Improve chain splitting in some corner cases.
Artem Belevich via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 17 13:42:34 PDT 2023
Author: Artem Belevich
Date: 2023-04-17T13:42:00-07:00
New Revision: faa631f939e43260deb707cffbeda4cda341f350
URL: https://github.com/llvm/llvm-project/commit/faa631f939e43260deb707cffbeda4cda341f350
DIFF: https://github.com/llvm/llvm-project/commit/faa631f939e43260deb707cffbeda4cda341f350.diff
LOG: [LSV] Improve chain splitting in some corner cases.
Currently we happen to split a chain of 12xi8 accesses into 6xi8 + 6xi8, which
produces rather suboptimal code.
This change attempts to split-off non-multiples of 4bytes at the end and if that
does not work, splits on the smaller power-of-2 boundary.
Differential Revision: https://reviews.llvm.org/D147976
Added:
llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i8.ll
Modified:
llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 6d8fbd1c2e2cd..5633d4c4254c3 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -665,14 +665,16 @@ Vectorizer::splitOddVectorElts(ArrayRef<Instruction *> Chain,
unsigned ElementSizeBits) {
unsigned ElementSizeBytes = ElementSizeBits / 8;
unsigned SizeBytes = ElementSizeBytes * Chain.size();
- unsigned NumLeft = (SizeBytes - (SizeBytes % 4)) / ElementSizeBytes;
- if (NumLeft == Chain.size()) {
- if ((NumLeft & 1) == 0)
- NumLeft /= 2; // Split even in half
- else
- --NumLeft; // Split off last element
- } else if (NumLeft == 0)
+ unsigned LeftBytes = (SizeBytes - (SizeBytes % 4));
+ // If we're already a multiple of 4 bytes or the whole chain is shorter than 4
+ // bytes, then try splitting down on power-of-2 boundary.
+ if (LeftBytes == SizeBytes || LeftBytes == 0)
+ LeftBytes = PowerOf2Ceil(SizeBytes) / 2;
+ unsigned NumLeft = LeftBytes / ElementSizeBytes;
+ if (NumLeft == 0)
NumLeft = 1;
+ LLVM_DEBUG(dbgs() << "LSV: Splitting the chain into " << NumLeft << "+"
+ << Chain.size() - NumLeft << " elements\n");
return std::make_pair(Chain.slice(0, NumLeft), Chain.slice(NumLeft));
}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i8.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i8.ll
new file mode 100644
index 0000000000000..387d6789a4c46
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i8.ll
@@ -0,0 +1,309 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
+
+; Vectorize and emit valid code (Issue #54896).
+
+define void @int8x3a2(ptr nocapture align 2 %ptr) {
+ %ptr0 = getelementptr i8, ptr %ptr, i64 0
+ %ptr1 = getelementptr i8, ptr %ptr, i64 1
+ %ptr2 = getelementptr i8, ptr %ptr, i64 2
+
+ %l0 = load i8, ptr %ptr0, align 2
+ %l1 = load i8, ptr %ptr1, align 1
+ %l2 = load i8, ptr %ptr2, align 2
+
+ store i8 %l2, ptr %ptr0, align 2
+ store i8 %l1, ptr %ptr1, align 1
+ store i8 %l0, ptr %ptr2, align 2
+
+ ret void
+
+; CHECK-LABEL: @int8x3a2
+; CHECK-DAG: load <2 x i8>
+; CHECK-DAG: load i8
+; CHECK-DAG: store <2 x i8>
+; CHECK-DAG: store i8
+}
+
+define void @int8x3a4(ptr nocapture align 4 %ptr) {
+ %ptr0 = getelementptr i8, ptr %ptr, i64 0
+ %ptr1 = getelementptr i8, ptr %ptr, i64 1
+ %ptr2 = getelementptr i8, ptr %ptr, i64 2
+
+ %l0 = load i8, ptr %ptr0, align 4
+ %l1 = load i8, ptr %ptr1, align 1
+ %l2 = load i8, ptr %ptr2, align 2
+
+ store i8 %l2, ptr %ptr0, align 2
+ store i8 %l1, ptr %ptr1, align 1
+ store i8 %l0, ptr %ptr2, align 4
+
+ ret void
+
+; CHECK-LABEL: @int8x3a4
+; CHECK: load <3 x i8>
+; CHECK: store <3 x i8>
+}
+
+define void @int8x12a4(ptr nocapture align 4 %ptr) {
+ %ptr0 = getelementptr i8, ptr %ptr, i64 0
+ %ptr1 = getelementptr i8, ptr %ptr, i64 1
+ %ptr2 = getelementptr i8, ptr %ptr, i64 2
+ %ptr3 = getelementptr i8, ptr %ptr, i64 3
+ %ptr4 = getelementptr i8, ptr %ptr, i64 4
+ %ptr5 = getelementptr i8, ptr %ptr, i64 5
+ %ptr6 = getelementptr i8, ptr %ptr, i64 6
+ %ptr7 = getelementptr i8, ptr %ptr, i64 7
+ %ptr8 = getelementptr i8, ptr %ptr, i64 8
+ %ptr9 = getelementptr i8, ptr %ptr, i64 9
+ %ptra = getelementptr i8, ptr %ptr, i64 10
+ %ptrb = getelementptr i8, ptr %ptr, i64 11
+
+ %l0 = load i8, ptr %ptr0, align 4
+ %l1 = load i8, ptr %ptr1, align 1
+ %l2 = load i8, ptr %ptr2, align 2
+ %l3 = load i8, ptr %ptr3, align 1
+ %l4 = load i8, ptr %ptr4, align 4
+ %l5 = load i8, ptr %ptr5, align 1
+ %l6 = load i8, ptr %ptr6, align 2
+ %l7 = load i8, ptr %ptr7, align 1
+ %l8 = load i8, ptr %ptr8, align 4
+ %l9 = load i8, ptr %ptr9, align 1
+ %la = load i8, ptr %ptra, align 2
+ %lb = load i8, ptr %ptrb, align 1
+
+ store i8 %lb, ptr %ptr0, align 4
+ store i8 %la, ptr %ptr1, align 1
+ store i8 %l9, ptr %ptr2, align 2
+ store i8 %l8, ptr %ptr3, align 1
+ store i8 %l7, ptr %ptr4, align 4
+ store i8 %l6, ptr %ptr5, align 1
+ store i8 %l5, ptr %ptr6, align 2
+ store i8 %l4, ptr %ptr7, align 1
+ store i8 %l3, ptr %ptr8, align 4
+ store i8 %l2, ptr %ptr9, align 1
+ store i8 %l1, ptr %ptra, align 2
+ store i8 %l0, ptr %ptrb, align 1
+
+ ret void
+
+; CHECK-LABEL: @int8x12a4
+; CHECK: load <4 x i8>
+; CHECK: load <4 x i8>
+; CHECK: load <4 x i8>
+; CHECK: store <4 x i8>
+; CHECK: store <4 x i8>
+; CHECK: store <4 x i8>
+}
+
+
+define void @int8x16a4(ptr nocapture align 4 %ptr) {
+ %ptr0 = getelementptr i8, ptr %ptr, i64 0
+ %ptr1 = getelementptr i8, ptr %ptr, i64 1
+ %ptr2 = getelementptr i8, ptr %ptr, i64 2
+ %ptr3 = getelementptr i8, ptr %ptr, i64 3
+ %ptr4 = getelementptr i8, ptr %ptr, i64 4
+ %ptr5 = getelementptr i8, ptr %ptr, i64 5
+ %ptr6 = getelementptr i8, ptr %ptr, i64 6
+ %ptr7 = getelementptr i8, ptr %ptr, i64 7
+ %ptr8 = getelementptr i8, ptr %ptr, i64 8
+ %ptr9 = getelementptr i8, ptr %ptr, i64 9
+ %ptra = getelementptr i8, ptr %ptr, i64 10
+ %ptrb = getelementptr i8, ptr %ptr, i64 11
+ %ptrc = getelementptr i8, ptr %ptr, i64 12
+ %ptrd = getelementptr i8, ptr %ptr, i64 13
+ %ptre = getelementptr i8, ptr %ptr, i64 14
+ %ptrf = getelementptr i8, ptr %ptr, i64 15
+
+ %l0 = load i8, ptr %ptr0, align 4
+ %l1 = load i8, ptr %ptr1, align 1
+ %l2 = load i8, ptr %ptr2, align 2
+ %l3 = load i8, ptr %ptr3, align 1
+ %l4 = load i8, ptr %ptr4, align 4
+ %l5 = load i8, ptr %ptr5, align 1
+ %l6 = load i8, ptr %ptr6, align 2
+ %l7 = load i8, ptr %ptr7, align 1
+ %l8 = load i8, ptr %ptr8, align 4
+ %l9 = load i8, ptr %ptr9, align 1
+ %la = load i8, ptr %ptra, align 2
+ %lb = load i8, ptr %ptrb, align 1
+ %lc = load i8, ptr %ptrc, align 4
+ %ld = load i8, ptr %ptrd, align 1
+ %le = load i8, ptr %ptre, align 2
+ %lf = load i8, ptr %ptrf, align 1
+
+ store i8 %lf, ptr %ptrc, align 4
+ store i8 %le, ptr %ptrd, align 1
+ store i8 %ld, ptr %ptre, align 2
+ store i8 %lc, ptr %ptrf, align 1
+ store i8 %lb, ptr %ptr0, align 4
+ store i8 %la, ptr %ptr1, align 1
+ store i8 %l9, ptr %ptr2, align 2
+ store i8 %l8, ptr %ptr3, align 1
+ store i8 %l7, ptr %ptr4, align 4
+ store i8 %l6, ptr %ptr5, align 1
+ store i8 %l5, ptr %ptr6, align 2
+ store i8 %l4, ptr %ptr7, align 1
+ store i8 %l3, ptr %ptr8, align 4
+ store i8 %l2, ptr %ptr9, align 1
+ store i8 %l1, ptr %ptra, align 2
+ store i8 %l0, ptr %ptrb, align 1
+
+ ret void
+
+; CHECK-LABEL: @int8x16a4
+; CHECK: load <4 x i8>
+; CHECK: load <4 x i8>
+; CHECK: load <4 x i8>
+; CHECK: load <4 x i8>
+; CHECK: store <4 x i8>
+; CHECK: store <4 x i8>
+; CHECK: store <4 x i8>
+; CHECK: store <4 x i8>
+}
+
+define void @int8x8a8(ptr nocapture align 8 %ptr) {
+ %ptr0 = getelementptr i8, ptr %ptr, i64 0
+ %ptr1 = getelementptr i8, ptr %ptr, i64 1
+ %ptr2 = getelementptr i8, ptr %ptr, i64 2
+ %ptr3 = getelementptr i8, ptr %ptr, i64 3
+ %ptr4 = getelementptr i8, ptr %ptr, i64 4
+ %ptr5 = getelementptr i8, ptr %ptr, i64 5
+ %ptr6 = getelementptr i8, ptr %ptr, i64 6
+ %ptr7 = getelementptr i8, ptr %ptr, i64 7
+
+ %l0 = load i8, ptr %ptr0, align 8
+ %l1 = load i8, ptr %ptr1, align 1
+ %l2 = load i8, ptr %ptr2, align 2
+ %l3 = load i8, ptr %ptr3, align 1
+ %l4 = load i8, ptr %ptr4, align 4
+ %l5 = load i8, ptr %ptr5, align 1
+ %l6 = load i8, ptr %ptr6, align 2
+ %l7 = load i8, ptr %ptr7, align 1
+
+ store i8 %l7, ptr %ptr0, align 8
+ store i8 %l6, ptr %ptr1, align 1
+ store i8 %l5, ptr %ptr2, align 2
+ store i8 %l4, ptr %ptr3, align 1
+ store i8 %l3, ptr %ptr4, align 4
+ store i8 %l2, ptr %ptr5, align 1
+ store i8 %l1, ptr %ptr6, align 2
+ store i8 %l0, ptr %ptr7, align 1
+
+ ret void
+
+; CHECK-LABEL: @int8x8a8
+; CHECK: load <8 x i8>
+; CHECK: store <8 x i8>
+}
+
+define void @int8x12a8(ptr nocapture align 8 %ptr) {
+ %ptr0 = getelementptr i8, ptr %ptr, i64 0
+ %ptr1 = getelementptr i8, ptr %ptr, i64 1
+ %ptr2 = getelementptr i8, ptr %ptr, i64 2
+ %ptr3 = getelementptr i8, ptr %ptr, i64 3
+ %ptr4 = getelementptr i8, ptr %ptr, i64 4
+ %ptr5 = getelementptr i8, ptr %ptr, i64 5
+ %ptr6 = getelementptr i8, ptr %ptr, i64 6
+ %ptr7 = getelementptr i8, ptr %ptr, i64 7
+ %ptr8 = getelementptr i8, ptr %ptr, i64 8
+ %ptr9 = getelementptr i8, ptr %ptr, i64 9
+ %ptra = getelementptr i8, ptr %ptr, i64 10
+ %ptrb = getelementptr i8, ptr %ptr, i64 11
+
+ %l0 = load i8, ptr %ptr0, align 8
+ %l1 = load i8, ptr %ptr1, align 1
+ %l2 = load i8, ptr %ptr2, align 2
+ %l3 = load i8, ptr %ptr3, align 1
+ %l4 = load i8, ptr %ptr4, align 4
+ %l5 = load i8, ptr %ptr5, align 1
+ %l6 = load i8, ptr %ptr6, align 2
+ %l7 = load i8, ptr %ptr7, align 1
+ %l8 = load i8, ptr %ptr8, align 8
+ %l9 = load i8, ptr %ptr9, align 1
+ %la = load i8, ptr %ptra, align 2
+ %lb = load i8, ptr %ptrb, align 1
+
+ store i8 %lb, ptr %ptr0, align 8
+ store i8 %la, ptr %ptr1, align 1
+ store i8 %l9, ptr %ptr2, align 2
+ store i8 %l8, ptr %ptr3, align 1
+ store i8 %l7, ptr %ptr4, align 4
+ store i8 %l6, ptr %ptr5, align 1
+ store i8 %l5, ptr %ptr6, align 2
+ store i8 %l4, ptr %ptr7, align 1
+ store i8 %l3, ptr %ptr8, align 8
+ store i8 %l2, ptr %ptr9, align 1
+ store i8 %l1, ptr %ptra, align 2
+ store i8 %l0, ptr %ptrb, align 1
+
+ ret void
+
+; CHECK-LABEL: @int8x12a8
+; CHECK-DAG: load <8 x i8>
+; CHECK-DAG: load <4 x i8>
+; CHECK-DAG: store <8 x i8>
+; CHECK-DAG: store <4 x i8>
+}
+
+
+define void @int8x16a8(ptr nocapture align 8 %ptr) {
+ %ptr0 = getelementptr i8, ptr %ptr, i64 0
+ %ptr1 = getelementptr i8, ptr %ptr, i64 1
+ %ptr2 = getelementptr i8, ptr %ptr, i64 2
+ %ptr3 = getelementptr i8, ptr %ptr, i64 3
+ %ptr4 = getelementptr i8, ptr %ptr, i64 4
+ %ptr5 = getelementptr i8, ptr %ptr, i64 5
+ %ptr6 = getelementptr i8, ptr %ptr, i64 6
+ %ptr7 = getelementptr i8, ptr %ptr, i64 7
+ %ptr8 = getelementptr i8, ptr %ptr, i64 8
+ %ptr9 = getelementptr i8, ptr %ptr, i64 9
+ %ptra = getelementptr i8, ptr %ptr, i64 10
+ %ptrb = getelementptr i8, ptr %ptr, i64 11
+ %ptrc = getelementptr i8, ptr %ptr, i64 12
+ %ptrd = getelementptr i8, ptr %ptr, i64 13
+ %ptre = getelementptr i8, ptr %ptr, i64 14
+ %ptrf = getelementptr i8, ptr %ptr, i64 15
+
+ %l0 = load i8, ptr %ptr0, align 8
+ %l1 = load i8, ptr %ptr1, align 1
+ %l2 = load i8, ptr %ptr2, align 2
+ %l3 = load i8, ptr %ptr3, align 1
+ %l4 = load i8, ptr %ptr4, align 4
+ %l5 = load i8, ptr %ptr5, align 1
+ %l6 = load i8, ptr %ptr6, align 2
+ %l7 = load i8, ptr %ptr7, align 1
+ %l8 = load i8, ptr %ptr8, align 8
+ %l9 = load i8, ptr %ptr9, align 1
+ %la = load i8, ptr %ptra, align 2
+ %lb = load i8, ptr %ptrb, align 1
+ %lc = load i8, ptr %ptrc, align 4
+ %ld = load i8, ptr %ptrd, align 1
+ %le = load i8, ptr %ptre, align 2
+ %lf = load i8, ptr %ptrf, align 1
+
+ store i8 %lf, ptr %ptr0, align 8
+ store i8 %le, ptr %ptr1, align 1
+ store i8 %ld, ptr %ptr2, align 2
+ store i8 %lc, ptr %ptr3, align 1
+ store i8 %lb, ptr %ptr4, align 4
+ store i8 %la, ptr %ptr5, align 1
+ store i8 %l9, ptr %ptr6, align 2
+ store i8 %l8, ptr %ptr7, align 1
+ store i8 %l7, ptr %ptr8, align 8
+ store i8 %l6, ptr %ptr9, align 1
+ store i8 %l5, ptr %ptra, align 2
+ store i8 %l4, ptr %ptrb, align 1
+ store i8 %l3, ptr %ptrc, align 4
+ store i8 %l2, ptr %ptrd, align 1
+ store i8 %l1, ptr %ptre, align 2
+ store i8 %l0, ptr %ptrf, align 1
+
+ ret void
+
+; CHECK-LABEL: @int8x16a8
+; CHECK: load <8 x i8>
+; CHECK: load <8 x i8>
+; CHECK: store <8 x i8>
+; CHECK: store <8 x i8>
+}
More information about the llvm-commits
mailing list