[llvm] [AArch64][GlobalISel] Combine Shuffles of G_CONCAT_VECTORS (PR #87489)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 10 13:23:33 PDT 2024
https://github.com/chuongg3 updated https://github.com/llvm/llvm-project/pull/87489
>From 7c841b82b6779ccf2a2c03909f50e6021429f7b9 Mon Sep 17 00:00:00 2001
From: Tuan Chuong Goh <chuong.goh at arm.com>
Date: Fri, 5 Apr 2024 17:42:02 +0000
Subject: [PATCH 1/4] [AArch64][GlobalISel] Pre-commit tests for Combine
Shuffle of G_CONCAT_VECTORS
---
.../GlobalISel/combine-shufflevector.mir | 213 ++++++++++++++++++
1 file changed, 213 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-shufflevector.mir
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-shufflevector.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-shufflevector.mir
new file mode 100644
index 00000000000000..9dd95efbc5b134
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-shufflevector.mir
@@ -0,0 +1,213 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -o - -mtriple=aarch64-unknown-unknown -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s | FileCheck %s
+
+---
+name: shuffle_concat_1
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $x0, $x1, $x2, $x3
+
+ ; CHECK-LABEL: name: shuffle_concat_1
+ ; CHECK: liveins: $x0, $x1, $x2, $x3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %p2:_(p0) = COPY $x1
+ ; CHECK-NEXT: %p3:_(p0) = COPY $x2
+ ; CHECK-NEXT: %p4:_(p0) = COPY $x3
+ ; CHECK-NEXT: %a:_(<4 x s8>) = G_LOAD %p4(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %b:_(<4 x s8>) = G_LOAD %p3(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %c:_(<4 x s8>) = G_LOAD %p2(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), %b(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
+ ; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %c(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
+ ; CHECK-NEXT: %z:_(<16 x s8>) = G_SHUFFLE_VECTOR [[CONCAT_VECTORS]](<16 x s8>), [[CONCAT_VECTORS1]], shufflemask(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, undef, undef, undef, undef)
+ ; CHECK-NEXT: $q0 = COPY %z(<16 x s8>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %p1:_(p0) = COPY $x0
+ %p2:_(p0) = COPY $x1
+ %p3:_(p0) = COPY $x2
+ %p4:_(p0) = COPY $x3
+
+ %ImpDef:_(<4 x s8>) = G_IMPLICIT_DEF
+ %a:_(<4 x s8>) = G_LOAD %p4:_(p0) :: (load (<4 x s8>))
+ %b:_(<4 x s8>) = G_LOAD %p3:_(p0) :: (load (<4 x s8>))
+ %c:_(<4 x s8>) = G_LOAD %p2:_(p0) :: (load (<4 x s8>))
+ %d:_(<4 x s8>) = G_LOAD %p1:_(p0) :: (load (<4 x s8>))
+
+ %x:_(<16 x s8>) = G_SHUFFLE_VECTOR %a:_(<4 x s8>), %b:_, shufflemask(0, 1, 2, 3, 4, 5, 6, 7, undef, undef, undef, undef, undef, undef, undef, undef)
+ %y:_(<16 x s8>) = G_SHUFFLE_VECTOR %c:_(<4 x s8>), %d:_, shufflemask(0, 1, 2, 3, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef)
+ %z:_(<16 x s8>) = G_SHUFFLE_VECTOR %x:_(<16 x s8>), %y:_, shufflemask(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, -1, -1, -1, -1)
+
+ $q0 = COPY %z(<16 x s8>)
+ RET_ReallyLR implicit $q0
+...
+
+---
+name: shuffle_concat_2
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $x0, $x1, $x2, $x3
+
+ ; CHECK-LABEL: name: shuffle_concat_2
+ ; CHECK: liveins: $x0, $x1, $x2, $x3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %p1:_(p0) = COPY $x0
+ ; CHECK-NEXT: %p2:_(p0) = COPY $x1
+ ; CHECK-NEXT: %p3:_(p0) = COPY $x2
+ ; CHECK-NEXT: %p4:_(p0) = COPY $x3
+ ; CHECK-NEXT: %ImpDef:_(<4 x s8>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: %a:_(<4 x s8>) = G_LOAD %p4(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %b:_(<4 x s8>) = G_LOAD %p3(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %c:_(<4 x s8>) = G_LOAD %p2(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %d:_(<4 x s8>) = G_LOAD %p1(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), %b(<4 x s8>), %ImpDef(<4 x s8>), %ImpDef(<4 x s8>)
+ ; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %c(<4 x s8>), %ImpDef(<4 x s8>), %ImpDef(<4 x s8>), %ImpDef(<4 x s8>)
+ ; CHECK-NEXT: %x:_(<16 x s8>) = G_SHUFFLE_VECTOR [[CONCAT_VECTORS]](<16 x s8>), [[CONCAT_VECTORS1]], shufflemask(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, undef, undef, undef, undef)
+ ; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %d(<4 x s8>), %ImpDef(<4 x s8>), %ImpDef(<4 x s8>), %ImpDef(<4 x s8>)
+ ; CHECK-NEXT: %z:_(<16 x s8>) = G_SHUFFLE_VECTOR %x(<16 x s8>), [[CONCAT_VECTORS2]], shufflemask(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19)
+ ; CHECK-NEXT: $q0 = COPY %z(<16 x s8>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %p1:_(p0) = COPY $x0
+ %p2:_(p0) = COPY $x1
+ %p3:_(p0) = COPY $x2
+ %p4:_(p0) = COPY $x3
+
+ %ImpDef:_(<4 x s8>) = G_IMPLICIT_DEF
+ %a:_(<4 x s8>) = G_LOAD %p4:_(p0) :: (load (<4 x s8>))
+ %b:_(<4 x s8>) = G_LOAD %p3:_(p0) :: (load (<4 x s8>))
+ %c:_(<4 x s8>) = G_LOAD %p2:_(p0) :: (load (<4 x s8>))
+ %d:_(<4 x s8>) = G_LOAD %p1:_(p0) :: (load (<4 x s8>))
+
+ %v:_(<16 x s8>) = G_SHUFFLE_VECTOR %a:_(<4 x s8>), %b:_, shufflemask(0, 1, 2, 3, 4, 5, 6, 7, undef, undef, undef, undef, undef, undef, undef, undef)
+ %w:_(<16 x s8>) = G_SHUFFLE_VECTOR %c:_(<4 x s8>), %ImpDef:_, shufflemask(0, 1, 2, 3, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef)
+ %x:_(<16 x s8>) = G_SHUFFLE_VECTOR %v:_(<16 x s8>), %w:_, shufflemask(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, undef, undef, undef, undef)
+ %y:_(<16 x s8>) = G_SHUFFLE_VECTOR %d:_(<4 x s8>), %ImpDef:_, shufflemask(0, 1, 2, 3, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef)
+ %z:_(<16 x s8>) = G_SHUFFLE_VECTOR %x:_(<16 x s8>), %y:_, shufflemask(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19)
+
+ $q0 = COPY %z(<16 x s8>)
+ RET_ReallyLR implicit $q0
+...
+
+---
+name: shuffle_concat_3
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $x0, $x1, $x2, $x3
+
+ ; CHECK-LABEL: name: shuffle_concat_3
+ ; CHECK: liveins: $x0, $x1, $x2, $x3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %p2:_(p0) = COPY $x1
+ ; CHECK-NEXT: %p3:_(p0) = COPY $x2
+ ; CHECK-NEXT: %p4:_(p0) = COPY $x3
+ ; CHECK-NEXT: %a:_(<4 x s8>) = G_LOAD %p4(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %b:_(<4 x s8>) = G_LOAD %p3(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %c:_(<4 x s8>) = G_LOAD %p2(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), %b(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
+ ; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %c(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
+ ; CHECK-NEXT: %z:_(<16 x s8>) = G_SHUFFLE_VECTOR [[CONCAT_VECTORS]](<16 x s8>), [[CONCAT_VECTORS1]], shufflemask(0, undef, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, undef, undef, undef, undef)
+ ; CHECK-NEXT: $q0 = COPY %z(<16 x s8>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %p1:_(p0) = COPY $x0
+ %p2:_(p0) = COPY $x1
+ %p3:_(p0) = COPY $x2
+ %p4:_(p0) = COPY $x3
+
+ %ImpDef:_(<4 x s8>) = G_IMPLICIT_DEF
+ %a:_(<4 x s8>) = G_LOAD %p4:_(p0) :: (load (<4 x s8>))
+ %b:_(<4 x s8>) = G_LOAD %p3:_(p0) :: (load (<4 x s8>))
+ %c:_(<4 x s8>) = G_LOAD %p2:_(p0) :: (load (<4 x s8>))
+ %d:_(<4 x s8>) = G_LOAD %p1:_(p0) :: (load (<4 x s8>))
+
+ %x:_(<16 x s8>) = G_SHUFFLE_VECTOR %a:_(<4 x s8>), %b:_, shufflemask(0, 1, 2, 3, 4, 5, 6, 7, undef, undef, undef, undef, undef, undef, undef, undef)
+ %y:_(<16 x s8>) = G_SHUFFLE_VECTOR %c:_(<4 x s8>), %d:_, shufflemask(0, 1, 2, 3, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef)
+ %z:_(<16 x s8>) = G_SHUFFLE_VECTOR %x:_(<16 x s8>), %y:_, shufflemask(0, -1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, -1, -1, -1, -1)
+
+ $q0 = COPY %z(<16 x s8>)
+ RET_ReallyLR implicit $q0
+...
+
+---
+name: shuffle_concat_4
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $x0, $x1, $x2, $x3
+
+ ; CHECK-LABEL: name: shuffle_concat_4
+ ; CHECK: liveins: $x0, $x1, $x2, $x3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %p2:_(p0) = COPY $x1
+ ; CHECK-NEXT: %p3:_(p0) = COPY $x2
+ ; CHECK-NEXT: %p4:_(p0) = COPY $x3
+ ; CHECK-NEXT: %a:_(<4 x s8>) = G_LOAD %p4(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %b:_(<4 x s8>) = G_LOAD %p3(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %c:_(<4 x s8>) = G_LOAD %p2(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), %b(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
+ ; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %c(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
+ ; CHECK-NEXT: %z:_(<16 x s8>) = G_SHUFFLE_VECTOR [[CONCAT_VECTORS]](<16 x s8>), [[CONCAT_VECTORS1]], shufflemask(0, 1, 2, 3, undef, undef, undef, undef, 16, 17, 18, 19, undef, undef, undef, undef)
+ ; CHECK-NEXT: $q0 = COPY %z(<16 x s8>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %p1:_(p0) = COPY $x0
+ %p2:_(p0) = COPY $x1
+ %p3:_(p0) = COPY $x2
+ %p4:_(p0) = COPY $x3
+
+ %ImpDef:_(<4 x s8>) = G_IMPLICIT_DEF
+ %a:_(<4 x s8>) = G_LOAD %p4:_(p0) :: (load (<4 x s8>))
+ %b:_(<4 x s8>) = G_LOAD %p3:_(p0) :: (load (<4 x s8>))
+ %c:_(<4 x s8>) = G_LOAD %p2:_(p0) :: (load (<4 x s8>))
+ %d:_(<4 x s8>) = G_LOAD %p1:_(p0) :: (load (<4 x s8>))
+
+ %x:_(<16 x s8>) = G_SHUFFLE_VECTOR %a:_(<4 x s8>), %b:_, shufflemask(0, 1, 2, 3, 4, 5, 6, 7, undef, undef, undef, undef, undef, undef, undef, undef)
+ %y:_(<16 x s8>) = G_SHUFFLE_VECTOR %c:_(<4 x s8>), %d:_, shufflemask(0, 1, 2, 3, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef)
+ %z:_(<16 x s8>) = G_SHUFFLE_VECTOR %x:_(<16 x s8>), %y:_, shufflemask(0, 1, 2, 3, -1, -1, -1, -1, 16, 17, 18, 19, -1, -1, -1, -1)
+
+ $q0 = COPY %z(<16 x s8>)
+ RET_ReallyLR implicit $q0
+...
+
+---
+name: shuffle_concat_5
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $x0, $x1, $x2, $x3
+
+ ; CHECK-LABEL: name: shuffle_concat_5
+ ; CHECK: liveins: $x0, $x1, $x2, $x3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %p2:_(p0) = COPY $x1
+ ; CHECK-NEXT: %p3:_(p0) = COPY $x2
+ ; CHECK-NEXT: %p4:_(p0) = COPY $x3
+ ; CHECK-NEXT: %a:_(<4 x s8>) = G_LOAD %p4(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %b:_(<4 x s8>) = G_LOAD %p3(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: %c:_(<4 x s8>) = G_LOAD %p2(p0) :: (load (<4 x s8>))
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
+ ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), %b(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
+ ; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %c(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
+ ; CHECK-NEXT: %z:_(<16 x s8>) = G_SHUFFLE_VECTOR [[CONCAT_VECTORS]](<16 x s8>), [[CONCAT_VECTORS1]], shufflemask(undef, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, undef, undef, undef, undef)
+ ; CHECK-NEXT: $q0 = COPY %z(<16 x s8>)
+ ; CHECK-NEXT: RET_ReallyLR implicit $q0
+ %p1:_(p0) = COPY $x0
+ %p2:_(p0) = COPY $x1
+ %p3:_(p0) = COPY $x2
+ %p4:_(p0) = COPY $x3
+
+ %ImpDef:_(<4 x s8>) = G_IMPLICIT_DEF
+ %a:_(<4 x s8>) = G_LOAD %p4:_(p0) :: (load (<4 x s8>))
+ %b:_(<4 x s8>) = G_LOAD %p3:_(p0) :: (load (<4 x s8>))
+ %c:_(<4 x s8>) = G_LOAD %p2:_(p0) :: (load (<4 x s8>))
+ %d:_(<4 x s8>) = G_LOAD %p1:_(p0) :: (load (<4 x s8>))
+
+ %x:_(<16 x s8>) = G_SHUFFLE_VECTOR %a:_(<4 x s8>), %b:_, shufflemask(0, 1, 2, 3, 4, 5, 6, 7, undef, undef, undef, undef, undef, undef, undef, undef)
+ %y:_(<16 x s8>) = G_SHUFFLE_VECTOR %c:_(<4 x s8>), %d:_, shufflemask(0, 1, 2, 3, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef, undef)
+ %z:_(<16 x s8>) = G_SHUFFLE_VECTOR %x:_(<16 x s8>), %y:_, shufflemask(-1, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, -1, -1, -1, -1)
+
+ $q0 = COPY %z(<16 x s8>)
+ RET_ReallyLR implicit $q0
+...
>From 5865a11a976be4d6fc851e75f12c6a4fac71e0d8 Mon Sep 17 00:00:00 2001
From: Tuan Chuong Goh <chuong.goh at arm.com>
Date: Wed, 3 Apr 2024 12:40:15 +0000
Subject: [PATCH 2/4] [AAAArch64][GlobalISel] Combine Shuffles of
G_CONCAT_VECTORS
Matches masks of G_SHUFFLE_VECTOR to see if they can be combined to
a G_CONCAT_VECTORS instruction
a = G_CONCAT_VECTORS x, y, undef, undef
b = G_CONCAT_VECTORS z, undef, undef, undef
c = G_SHUFFLE_VECTORS a, b, <0, 1, 4, undef>
===>
c = G_CONCAT_VECTORS x, y, z, undef
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 5 ++
.../include/llvm/Target/GlobalISel/Combine.td | 16 +++-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 73 +++++++++++++++++++
.../GlobalISel/combine-shufflevector.mir | 17 +----
4 files changed, 96 insertions(+), 15 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 3af32043391fec..4b8aec8e8a5dd6 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -237,6 +237,11 @@ class CombinerHelper {
/// or an implicit_def if \p Ops is empty.
void applyCombineConcatVectors(MachineInstr &MI, SmallVector<Register> &Ops);
+ bool matchCombineShuffleConcat(MachineInstr &MI, SmallVector<Register> &Ops);
+ /// Replace \p MI with a flattened build_vector with \p Ops
+ /// or an implicit_def if \p Ops is empty.
+ void applyCombineShuffleConcat(MachineInstr &MI, SmallVector<Register> &Ops);
+
/// Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS.
/// Returns true if MI changed.
///
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 778ff7e437eb50..8401e48c1bc9eb 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -1507,6 +1507,18 @@ def combine_concat_vector : GICombineRule<
[{ return Helper.matchCombineConcatVectors(*${root}, ${matchinfo}); }]),
(apply [{ Helper.applyCombineConcatVectors(*${root}, ${matchinfo}); }])>;
+// Combines Shuffles of Concats
+// a = G_CONCAT_VECTORS x, y, undef, undef
+// b = G_CONCAT_VECTORS z, undef, undef, undef
+// c = G_SHUFFLE_VECTORS a, b, <0, 1, 4, undef>
+// ===>
+// c = G_CONCAT_VECTORS x, y, z, undef
+def combine_shuffle_concat : GICombineRule<
+ (defs root:$root, concat_matchinfo:$matchinfo),
+ (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
+ [{ return Helper.matchCombineShuffleConcat(*${root}, ${matchinfo}); }]),
+ (apply [{ Helper.applyCombineShuffleConcat(*${root}, ${matchinfo}); }])>;
+
// match_extract_of_element must be the first!
def vector_ops_combines: GICombineGroup<[
match_extract_of_element_undef_vector,
@@ -1538,6 +1550,7 @@ extract_vector_element_build_vector_trunc8,
extract_vector_element_freeze
]>;
+
// FIXME: These should use the custom predicate feature once it lands.
def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
undef_to_negative_one,
@@ -1614,7 +1627,8 @@ def all_combines : GICombineGroup<[trivial_combines, vector_ops_combines,
and_or_disjoint_mask, fma_combines, fold_binop_into_select,
sub_add_reg, select_to_minmax, redundant_binop_in_equality,
fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
- combine_concat_vector, double_icmp_zero_and_or_combine, match_addos]>;
+ combine_concat_vector, double_icmp_zero_and_or_combine, match_addos,
+ combine_shuffle_concat]>;
// A combine group used to for prelegalizer combiners at -O0. The combines in
// this group have been selected based on experiments to balance code size and
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 719209e0edd5fb..9c18e0f136b715 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -303,6 +303,79 @@ void CombinerHelper::applyCombineConcatVectors(MachineInstr &MI,
replaceRegWith(MRI, DstReg, NewDstReg);
}
+bool CombinerHelper::matchCombineShuffleConcat(MachineInstr &MI,
+ SmallVector<Register> &Ops) {
+ ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
+ auto ConcatMI1 = dyn_cast<GConcatVectors>(
+ getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI));
+ auto ConcatMI2 = dyn_cast<GConcatVectors>(
+ getDefIgnoringCopies(MI.getOperand(2).getReg(), MRI));
+ if (!ConcatMI1 || !ConcatMI2)
+ return false;
+
+ // Check that the sources of the Concat instructions have the same type
+ if (MRI.getType(ConcatMI1->getSourceReg(0)) !=
+ MRI.getType(ConcatMI2->getSourceReg(0)))
+ return false;
+
+ LLT ConcatSrcTy = MRI.getType(ConcatMI1->getReg(1));
+ LLT ShuffleSrcTy1 = MRI.getType(MI.getOperand(1).getReg());
+ unsigned ConcatSrcNumElt = ConcatSrcTy.getNumElements();
+ for (unsigned i = 0; i < Mask.size(); i += ConcatSrcNumElt) {
+ // Check if the index takes a whole source register from G_CONCAT_VECTORS
+ // Assumes that all Sources of G_CONCAT_VECTORS are the same type
+ if (Mask[i] == -1) {
+ for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
+ if (i + j >= Mask.size())
+ return false;
+ if (Mask[i + j] != -1)
+ return false;
+ }
+ Ops.push_back(0);
+ } else if (Mask[i] % (int)ConcatSrcNumElt == 0) {
+ for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
+ if (i + j >= Mask.size())
+ return false;
+ if (Mask[i + j] != Mask[i] + (int)j)
+ return false;
+ }
+ // Retrieve the source register from its respective G_CONCAT_VECTORS
+ // instruction
+ if (Mask[i] < (int)ShuffleSrcTy1.getNumElements()) {
+ Ops.push_back(ConcatMI1->getSourceReg(Mask[i] / (int)ConcatSrcNumElt));
+ } else {
+ Ops.push_back(ConcatMI2->getSourceReg(Mask[i] / (int)ConcatSrcNumElt -
+ (int)ConcatMI1->getNumSources()));
+ }
+ } else {
+ return false;
+ }
+ }
+
+ if (Ops.size() == 0)
+ return false;
+ // Only deal with cases where G_CONCAT_VECTORS sources are all the same type
+ if ((Mask.size() - (Ops.size() * ConcatSrcNumElt)) %
+ MRI.getType(Ops[0]).getNumElements() !=
+ 0)
+ return false;
+ return true;
+}
+
+void CombinerHelper::applyCombineShuffleConcat(MachineInstr &MI,
+ SmallVector<Register> &Ops) {
+ LLT SrcTy = MRI.getType(Ops[0]);
+ Register UndefReg = Builder.buildUndef(SrcTy).getReg(0);
+
+ for (unsigned i = 0; i < Ops.size(); i++) {
+ if (Ops[i] == 0)
+ Ops[i] = UndefReg;
+ }
+
+ Builder.buildConcatVectors(MI.getOperand(0).getReg(), Ops);
+ MI.eraseFromParent();
+}
+
bool CombinerHelper::tryCombineShuffleVector(MachineInstr &MI) {
SmallVector<Register, 4> Ops;
if (matchCombineShuffleVector(MI, Ops)) {
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-shufflevector.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-shufflevector.mir
index 9dd95efbc5b134..0de989f8be75d7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-shufflevector.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-shufflevector.mir
@@ -18,9 +18,7 @@ body: |
; CHECK-NEXT: %b:_(<4 x s8>) = G_LOAD %p3(p0) :: (load (<4 x s8>))
; CHECK-NEXT: %c:_(<4 x s8>) = G_LOAD %p2(p0) :: (load (<4 x s8>))
; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), %b(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
- ; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %c(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
- ; CHECK-NEXT: %z:_(<16 x s8>) = G_SHUFFLE_VECTOR [[CONCAT_VECTORS]](<16 x s8>), [[CONCAT_VECTORS1]], shufflemask(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, undef, undef, undef, undef)
+ ; CHECK-NEXT: %z:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), %b(<4 x s8>), %c(<4 x s8>), [[DEF]](<4 x s8>)
; CHECK-NEXT: $q0 = COPY %z(<16 x s8>)
; CHECK-NEXT: RET_ReallyLR implicit $q0
%p1:_(p0) = COPY $x0
@@ -56,16 +54,11 @@ body: |
; CHECK-NEXT: %p2:_(p0) = COPY $x1
; CHECK-NEXT: %p3:_(p0) = COPY $x2
; CHECK-NEXT: %p4:_(p0) = COPY $x3
- ; CHECK-NEXT: %ImpDef:_(<4 x s8>) = G_IMPLICIT_DEF
; CHECK-NEXT: %a:_(<4 x s8>) = G_LOAD %p4(p0) :: (load (<4 x s8>))
; CHECK-NEXT: %b:_(<4 x s8>) = G_LOAD %p3(p0) :: (load (<4 x s8>))
; CHECK-NEXT: %c:_(<4 x s8>) = G_LOAD %p2(p0) :: (load (<4 x s8>))
; CHECK-NEXT: %d:_(<4 x s8>) = G_LOAD %p1(p0) :: (load (<4 x s8>))
- ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), %b(<4 x s8>), %ImpDef(<4 x s8>), %ImpDef(<4 x s8>)
- ; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %c(<4 x s8>), %ImpDef(<4 x s8>), %ImpDef(<4 x s8>), %ImpDef(<4 x s8>)
- ; CHECK-NEXT: %x:_(<16 x s8>) = G_SHUFFLE_VECTOR [[CONCAT_VECTORS]](<16 x s8>), [[CONCAT_VECTORS1]], shufflemask(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, undef, undef, undef, undef)
- ; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %d(<4 x s8>), %ImpDef(<4 x s8>), %ImpDef(<4 x s8>), %ImpDef(<4 x s8>)
- ; CHECK-NEXT: %z:_(<16 x s8>) = G_SHUFFLE_VECTOR %x(<16 x s8>), [[CONCAT_VECTORS2]], shufflemask(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19)
+ ; CHECK-NEXT: %z:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), %b(<4 x s8>), %c(<4 x s8>), %d(<4 x s8>)
; CHECK-NEXT: $q0 = COPY %z(<16 x s8>)
; CHECK-NEXT: RET_ReallyLR implicit $q0
%p1:_(p0) = COPY $x0
@@ -141,15 +134,11 @@ body: |
; CHECK: liveins: $x0, $x1, $x2, $x3
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: %p2:_(p0) = COPY $x1
- ; CHECK-NEXT: %p3:_(p0) = COPY $x2
; CHECK-NEXT: %p4:_(p0) = COPY $x3
; CHECK-NEXT: %a:_(<4 x s8>) = G_LOAD %p4(p0) :: (load (<4 x s8>))
- ; CHECK-NEXT: %b:_(<4 x s8>) = G_LOAD %p3(p0) :: (load (<4 x s8>))
; CHECK-NEXT: %c:_(<4 x s8>) = G_LOAD %p2(p0) :: (load (<4 x s8>))
; CHECK-NEXT: [[DEF:%[0-9]+]]:_(<4 x s8>) = G_IMPLICIT_DEF
- ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), %b(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
- ; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<16 x s8>) = G_CONCAT_VECTORS %c(<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>), [[DEF]](<4 x s8>)
- ; CHECK-NEXT: %z:_(<16 x s8>) = G_SHUFFLE_VECTOR [[CONCAT_VECTORS]](<16 x s8>), [[CONCAT_VECTORS1]], shufflemask(0, 1, 2, 3, undef, undef, undef, undef, 16, 17, 18, 19, undef, undef, undef, undef)
+ ; CHECK-NEXT: %z:_(<16 x s8>) = G_CONCAT_VECTORS %a(<4 x s8>), [[DEF]](<4 x s8>), %c(<4 x s8>), [[DEF]](<4 x s8>)
; CHECK-NEXT: $q0 = COPY %z(<16 x s8>)
; CHECK-NEXT: RET_ReallyLR implicit $q0
%p1:_(p0) = COPY $x0
>From a2567ae0cef6f405310b1f36b781356d02f5e747 Mon Sep 17 00:00:00 2001
From: Tuan Chuong Goh <chuong.goh at arm.com>
Date: Mon, 8 Apr 2024 16:52:15 +0000
Subject: [PATCH 3/4] fixup! [AAAArch64][GlobalISel] Combine Shuffles of
G_CONCAT_VECTORS
---
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 19 ++++++-------------
1 file changed, 6 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 9c18e0f136b715..21fd79fc32ec77 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -306,10 +306,10 @@ void CombinerHelper::applyCombineConcatVectors(MachineInstr &MI,
bool CombinerHelper::matchCombineShuffleConcat(MachineInstr &MI,
SmallVector<Register> &Ops) {
ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
- auto ConcatMI1 = dyn_cast<GConcatVectors>(
- getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI));
- auto ConcatMI2 = dyn_cast<GConcatVectors>(
- getDefIgnoringCopies(MI.getOperand(2).getReg(), MRI));
+ auto ConcatMI1 =
+ dyn_cast<GConcatVectors>(MRI.getVRegDef(MI.getOperand(1).getReg()));
+ auto ConcatMI2 =
+ dyn_cast<GConcatVectors>(MRI.getVRegDef(MI.getOperand(2).getReg()));
if (!ConcatMI1 || !ConcatMI2)
return false;
@@ -336,7 +336,7 @@ bool CombinerHelper::matchCombineShuffleConcat(MachineInstr &MI,
for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
if (i + j >= Mask.size())
return false;
- if (Mask[i + j] != Mask[i] + (int)j)
+ if (Mask[i + j] != Mask[i] + static_cast<int>(j))
return false;
}
// Retrieve the source register from its respective G_CONCAT_VECTORS
@@ -352,14 +352,7 @@ bool CombinerHelper::matchCombineShuffleConcat(MachineInstr &MI,
}
}
- if (Ops.size() == 0)
- return false;
- // Only deal with cases where G_CONCAT_VECTORS sources are all the same type
- if ((Mask.size() - (Ops.size() * ConcatSrcNumElt)) %
- MRI.getType(Ops[0]).getNumElements() !=
- 0)
- return false;
- return true;
+ return !Ops.empty();
}
void CombinerHelper::applyCombineShuffleConcat(MachineInstr &MI,
>From 998db05df5e943556ef2f72c448ff674515862a3 Mon Sep 17 00:00:00 2001
From: Tuan Chuong Goh <chuong.goh at arm.com>
Date: Wed, 10 Apr 2024 19:47:34 +0000
Subject: [PATCH 4/4] fixup! fixup! [AAAArch64][GlobalISel] Combine Shuffles of
G_CONCAT_VECTORS
---
llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 21fd79fc32ec77..55e27150d4f262 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -332,7 +332,7 @@ bool CombinerHelper::matchCombineShuffleConcat(MachineInstr &MI,
return false;
}
Ops.push_back(0);
- } else if (Mask[i] % (int)ConcatSrcNumElt == 0) {
+ } else if (Mask[i] % ConcatSrcNumElt == 0) {
for (unsigned j = 1; j < ConcatSrcNumElt; j++) {
if (i + j >= Mask.size())
return false;
@@ -341,11 +341,11 @@ bool CombinerHelper::matchCombineShuffleConcat(MachineInstr &MI,
}
// Retrieve the source register from its respective G_CONCAT_VECTORS
// instruction
- if (Mask[i] < (int)ShuffleSrcTy1.getNumElements()) {
- Ops.push_back(ConcatMI1->getSourceReg(Mask[i] / (int)ConcatSrcNumElt));
+ if (Mask[i] < ShuffleSrcTy1.getNumElements()) {
+ Ops.push_back(ConcatMI1->getSourceReg(Mask[i] / ConcatSrcNumElt));
} else {
- Ops.push_back(ConcatMI2->getSourceReg(Mask[i] / (int)ConcatSrcNumElt -
- (int)ConcatMI1->getNumSources()));
+ Ops.push_back(ConcatMI2->getSourceReg(Mask[i] / ConcatSrcNumElt -
+ ConcatMI1->getNumSources()));
}
} else {
return false;
@@ -358,11 +358,14 @@ bool CombinerHelper::matchCombineShuffleConcat(MachineInstr &MI,
void CombinerHelper::applyCombineShuffleConcat(MachineInstr &MI,
SmallVector<Register> &Ops) {
LLT SrcTy = MRI.getType(Ops[0]);
- Register UndefReg = Builder.buildUndef(SrcTy).getReg(0);
+ Register UndefReg = 0;
for (unsigned i = 0; i < Ops.size(); i++) {
- if (Ops[i] == 0)
+ if (Ops[i] == 0) {
+ if (UndefReg == 0)
+ UndefReg = Builder.buildUndef(SrcTy).getReg(0);
Ops[i] = UndefReg;
+ }
}
Builder.buildConcatVectors(MI.getOperand(0).getReg(), Ops);
More information about the llvm-commits
mailing list