[llvm] [InstCombine] Optimistically allow multiple shufflevector uses in foldOpPhi (PR #114278)
Matthias Braun via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 6 14:46:56 PST 2024
https://github.com/MatzeB updated https://github.com/llvm/llvm-project/pull/114278
>From 6f74d52044a376301f621cac98e71a50f4058a84 Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze at braunis.de>
Date: Tue, 19 Nov 2024 11:25:32 -0800
Subject: [PATCH 1/3] Allow multiple uses in foldOpIntoPhi for shufflevector
instructions
---
.../InstCombine/InstCombineInternal.h | 3 +-
.../InstCombine/InstCombineVectorOps.cpp | 2 +-
.../InstCombine/InstructionCombining.cpp | 31 +++--
.../InstCombine/vec_shuffle-phi-multiuse.ll | 115 ++++++++++++++++++
4 files changed, 138 insertions(+), 13 deletions(-)
create mode 100644 llvm/test/Transforms/InstCombine/vec_shuffle-phi-multiuse.ll
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index adbd9186c59c5a..d6092ca003d913 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -600,7 +600,8 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final
/// Given a binary operator, cast instruction, or select which has a PHI node
/// as operand #0, see if we can fold the instruction into the PHI (which is
/// only possible if all operands to the PHI are constants).
- Instruction *foldOpIntoPhi(Instruction &I, PHINode *PN);
+ Instruction *foldOpIntoPhi(Instruction &I, PHINode *PN,
+ bool AllowMultipleUses = false);
/// For a binary operator with 2 phi operands, try to hoist the binary
/// operation before the phi. This can result in fewer instructions in
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index ede89b099e8deb..9220d9c625758a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -2912,7 +2912,7 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
}
}
if (auto *PN = dyn_cast<PHINode>(LHS)) {
- if (Instruction *I = foldOpIntoPhi(SVI, PN))
+ if (Instruction *I = foldOpIntoPhi(SVI, PN, /*AllowMultipleUses=*/true))
return I;
}
}
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 32f2a30afad48f..18e0c0afa145b8 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1763,7 +1763,8 @@ static Value *simplifyInstructionWithPHI(Instruction &I, PHINode *PN,
return nullptr;
}
-Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
+Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN,
+ bool AllowMultipleUses) {
unsigned NumPHIValues = PN->getNumIncomingValues();
if (NumPHIValues == 0)
return nullptr;
@@ -1771,7 +1772,9 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
// We normally only transform phis with a single use. However, if a PHI has
// multiple uses and they are all the same operation, we can fold *all* of the
// uses into the PHI.
- if (!PN->hasOneUse()) {
+ bool OneUse = PN->hasOneUse();
+ bool IdenticalUsers = false;
+ if (!AllowMultipleUses && !OneUse) {
// Walk the use list for the instruction, comparing them to I.
for (User *U : PN->users()) {
Instruction *UI = cast<Instruction>(U);
@@ -1779,6 +1782,7 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
return nullptr;
}
// Otherwise, we can replace *all* users with the new PHI we form.
+ IdenticalUsers = true;
}
// Check that all operands are phi-translatable.
@@ -1890,17 +1894,22 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN) {
for (unsigned i = 0; i != NumPHIValues; ++i)
NewPN->addIncoming(NewPhiValues[i], PN->getIncomingBlock(i));
- for (User *U : make_early_inc_range(PN->users())) {
- Instruction *User = cast<Instruction>(U);
- if (User == &I)
- continue;
- replaceInstUsesWith(*User, NewPN);
- eraseInstFromFunction(*User);
+ if (IdenticalUsers) {
+ for (User *U : make_early_inc_range(PN->users())) {
+ Instruction *User = cast<Instruction>(U);
+ if (User == &I)
+ continue;
+ replaceInstUsesWith(*User, NewPN);
+ eraseInstFromFunction(*User);
+ }
+ OneUse = true;
}
- replaceAllDbgUsesWith(const_cast<PHINode &>(*PN),
- const_cast<PHINode &>(*NewPN),
- const_cast<PHINode &>(*PN), DT);
+ if (OneUse) {
+ replaceAllDbgUsesWith(const_cast<PHINode &>(*PN),
+ const_cast<PHINode &>(*NewPN),
+ const_cast<PHINode &>(*PN), DT);
+ }
return replaceInstUsesWith(I, NewPN);
}
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle-phi-multiuse.ll b/llvm/test/Transforms/InstCombine/vec_shuffle-phi-multiuse.ll
new file mode 100644
index 00000000000000..de99556e7c90bf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle-phi-multiuse.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -S -passes=instcombine | FileCheck %s
+
+define <4 x i16> @f0(i1 %c, ptr %p0, ptr %p1) {
+; CHECK-LABEL: define <4 x i16> @f0(
+; CHECK-SAME: i1 [[C:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[MERGE:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[LOAD0:%.*]] = load <4 x i16>, ptr [[P0]], align 16
+; CHECK-NEXT: [[LOAD1:%.*]] = load <4 x i16>, ptr [[P1]], align 16
+; CHECK-NEXT: [[TMP0:%.*]] = sub <4 x i16> [[LOAD0]], [[LOAD1]]
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[SUB:%.*]] = phi <4 x i16> [ <i16 -87, i16 327, i16 51, i16 755>, %[[ENTRY]] ], [ [[TMP0]], %[[THEN]] ]
+; CHECK-NEXT: ret <4 x i16> [[SUB]]
+;
+entry:
+ br i1 %c, label %then, label %merge
+
+then:
+ %load0 = load <4 x i16>, ptr %p0, align 16
+ %load1 = load <4 x i16>, ptr %p1, align 16
+ %interleave = shufflevector <4 x i16> %load0, <4 x i16> %load1, <8 x i32> <i32 0, i32 7, i32 1, i32 6, i32 2, i32 5, i32 3, i32 4>
+ br label %merge
+
+merge:
+ %phi = phi <8 x i16> [<i16 1, i16 22, i16 333, i16 4, i16 55, i16 6, i16 777, i16 88>, %entry], [%interleave, %then]
+ %shuf0 = shufflevector <8 x i16> %phi, <8 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %shuf1 = shufflevector <8 x i16> %phi, <8 x i16> poison, <4 x i32> <i32 7, i32 5, i32 3, i32 1>
+ %sub = sub <4 x i16> %shuf0, %shuf1
+ ret <4 x i16> %sub
+}
+
+define void @deinterleave_interleave(ptr %p_begin, ptr %p_end, ptr %out) {
+; CHECK-LABEL: define void @deinterleave_interleave(
+; CHECK-SAME: ptr [[P_BEGIN:%.*]], ptr [[P_END:%.*]], ptr [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[ACC:%.*]] = phi <4 x float> [ zeroinitializer, %[[ENTRY]] ], [ [[SUM_LOWS:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ODDS:%.*]] = phi <4 x float> [ zeroinitializer, %[[ENTRY]] ], [ [[SUM_HIGHS:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[P_BEGIN]], %[[ENTRY]] ], [ [[P_INC:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VAL:%.*]] = load <4 x i8>, ptr [[P]], align 4
+; CHECK-NEXT: [[HIGHS:%.*]] = ashr <4 x i8> [[VAL]], splat (i8 4)
+; CHECK-NEXT: [[LOWS:%.*]] = and <4 x i8> [[VAL]], splat (i8 15)
+; CHECK-NEXT: [[HIGHS_F:%.*]] = sitofp <4 x i8> [[HIGHS]] to <4 x float>
+; CHECK-NEXT: [[LOWS_F:%.*]] = uitofp nneg <4 x i8> [[LOWS]] to <4 x float>
+; CHECK-NEXT: [[SUM_LOWS]] = fadd <4 x float> [[ACC]], [[LOWS_F]]
+; CHECK-NEXT: [[SUM_HIGHS]] = fadd <4 x float> [[ODDS]], [[HIGHS_F]]
+; CHECK-NEXT: [[P_INC]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[P_INC]], [[P_END]]
+; CHECK-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[INTERLEAVE:%.*]] = shufflevector <4 x float> [[SUM_LOWS]], <4 x float> [[SUM_HIGHS]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT: store <8 x float> [[INTERLEAVE]], ptr [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %acc = phi <8 x float> [ zeroinitializer, %entry ], [ %interleave, %loop ]
+ %p = phi ptr [%p_begin, %entry ], [%p_inc, %loop]
+
+ %evens = shufflevector <8 x float> %acc, <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %odds = shufflevector <8 x float> %acc, <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+
+ %val = load <4 x i8>, ptr %p, align 4
+ %highs = ashr <4 x i8> %val, <i8 4, i8 4, i8 4, i8 4>
+ %lows = and <4 x i8> %val, <i8 15, i8 15, i8 15, i8 15>
+
+ %highs_f = sitofp <4 x i8> %highs to <4 x float>
+ %lows_f = sitofp <4 x i8> %lows to <4 x float>
+
+ %sum_lows = fadd <4 x float> %evens, %lows_f
+ %sum_highs = fadd <4 x float> %odds, %highs_f
+
+ %interleave = shufflevector <4 x float> %sum_lows, <4 x float> %sum_highs, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+
+ %p_inc = getelementptr inbounds i8, ptr %p, i32 4
+ %c = icmp eq ptr %p_inc, %p_end
+ br i1 %c, label %exit, label %loop
+
+exit:
+ store <8 x float> %interleave, ptr %out, align 4
+ ret void
+}
+
+define <4 x i16> @f1(i1 %c, ptr %p) {
+; CHECK-LABEL: define <4 x i16> @f1(
+; CHECK-SAME: i1 [[C:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[MERGE:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: store i32 42, ptr [[P]], align 4
+; CHECK-NEXT: br label %[[MERGE]]
+; CHECK: [[MERGE]]:
+; CHECK-NEXT: [[XOR:%.*]] = phi <4 x i16> [ <i16 3, i16 346, i16 undef, i16 undef>, %[[ENTRY]] ], [ <i16 7, i16 74, i16 undef, i16 undef>, %[[THEN]] ]
+; CHECK-NEXT: ret <4 x i16> [[XOR]]
+;
+entry:
+ br i1 %c, label %then, label %merge
+
+then:
+ store i32 42, ptr %p, align 4
+ br label %merge
+
+merge:
+ %phi = phi <4 x i16> [<i16 1, i16 22, i16 333, i16 4>, %entry], [<i16 555, i16 6, i16 77, i16 8>, %then]
+ %shuf0 = shufflevector <4 x i16> %phi, <4 x i16> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+ %add1 = add <4 x i16> %phi, <i16 1, i16 1, i16 1, i16 1>
+ %xor = xor <4 x i16> %shuf0, %add1
+ ret <4 x i16> %xor
+}
>From a48da4312798e7d689d544a6329ebc7139b6f04f Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze at braunis.de>
Date: Fri, 6 Dec 2024 13:39:54 -0800
Subject: [PATCH 2/3] Require all phi inputs to simplify when there are
multiple users
---
llvm/lib/Transforms/InstCombine/InstructionCombining.cpp | 3 +++
1 file changed, 3 insertions(+)
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 18e0c0afa145b8..e3e6933a81df9f 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1833,6 +1833,9 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN,
continue;
}
+ if (!OneUse)
+ return nullptr;
+
if (SeenNonSimplifiedInVal)
return nullptr; // More than one non-simplified value.
SeenNonSimplifiedInVal = true;
>From 8696cf6306bb312a49fb8c0b5d65b63c063b6554 Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze at braunis.de>
Date: Fri, 6 Dec 2024 14:35:42 -0800
Subject: [PATCH 3/3] adjust test to rebase
---
llvm/lib/Transforms/InstCombine/InstructionCombining.cpp | 2 +-
llvm/test/Transforms/InstCombine/vec_shuffle-phi-multiuse.ll | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index c5927db81aaf9e..e04ea57934e188 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1833,7 +1833,7 @@ Instruction *InstCombinerImpl::foldOpIntoPhi(Instruction &I, PHINode *PN,
continue;
}
- if (!OneUse)
+ if (!OneUse && !IdenticalUsers)
return nullptr;
if (SeenNonSimplifiedInVal)
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle-phi-multiuse.ll b/llvm/test/Transforms/InstCombine/vec_shuffle-phi-multiuse.ll
index de99556e7c90bf..44b25bb6fd727a 100644
--- a/llvm/test/Transforms/InstCombine/vec_shuffle-phi-multiuse.ll
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle-phi-multiuse.ll
@@ -48,7 +48,7 @@ define void @deinterleave_interleave(ptr %p_begin, ptr %p_end, ptr %out) {
; CHECK-NEXT: [[LOWS_F:%.*]] = uitofp nneg <4 x i8> [[LOWS]] to <4 x float>
; CHECK-NEXT: [[SUM_LOWS]] = fadd <4 x float> [[ACC]], [[LOWS_F]]
; CHECK-NEXT: [[SUM_HIGHS]] = fadd <4 x float> [[ODDS]], [[HIGHS_F]]
-; CHECK-NEXT: [[P_INC]] = getelementptr inbounds i8, ptr [[P]], i64 4
+; CHECK-NEXT: [[P_INC]] = getelementptr inbounds nuw i8, ptr [[P]], i64 4
; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[P_INC]], [[P_END]]
; CHECK-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
More information about the llvm-commits
mailing list