[llvm] f738ab9 - [DAGCombiner] `visitFREEZE()`: allow multiple maybe-poison operands for `BUILD_VECTOR`
Roman Lebedev via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 22 15:27:13 PST 2022
Author: Roman Lebedev
Date: 2022-12-23T02:26:36+03:00
New Revision: f738ab9075f838dd4365adf3a92ca1acced114d7
URL: https://github.com/llvm/llvm-project/commit/f738ab9075f838dd4365adf3a92ca1acced114d7
DIFF: https://github.com/llvm/llvm-project/commit/f738ab9075f838dd4365adf3a92ca1acced114d7.diff
LOG: [DAGCombiner] `visitFREEZE()`: allow multiple maybe-poison operands for `BUILD_VECTOR`
Added:
Modified:
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
llvm/test/CodeGen/X86/freeze-vector.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 71341f058b10..3076be8292ed 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -14246,40 +14246,49 @@ SDValue DAGCombiner::visitFREEZE(SDNode *N) {
if (DAG.isGuaranteedNotToBeUndefOrPoison(N0, /*PoisonOnly*/ false))
return N0;
- // Fold freeze(op(x,y,z)) -> op(freeze(x),y,z).
+ // Fold freeze(op(x, ...)) -> op(freeze(x), ...).
// Try to push freeze through instructions that propagate but don't produce
// poison as far as possible. If an operand of freeze follows three
// conditions 1) one-use, 2) does not produce poison, and 3) has all but one
- // guaranteed-non-poison operands then push the freeze through to the one
- // operand that is not guaranteed non-poison.
+ // guaranteed-non-poison operands (or is a BUILD_VECTOR or similar) then push
+ // the freeze through to the operands that are not guaranteed non-poison.
if (DAG.canCreateUndefOrPoison(N0, /*PoisonOnly*/ false,
/*ConsiderFlags*/ false) ||
N0->getNumValues() != 1 || !N0->hasOneUse())
return SDValue();
- SDValue MaybePoisonOperand;
+ bool AllowMultipleMaybePoisonOperands = N0.getOpcode() == ISD::BUILD_VECTOR;
+
+ SmallSetVector<SDValue, 8> MaybePoisonOperands;
for (SDValue Op : N0->ops()) {
if (DAG.isGuaranteedNotToBeUndefOrPoison(Op, /*PoisonOnly*/ false,
/*Depth*/ 1))
continue;
- if (!MaybePoisonOperand || MaybePoisonOperand == Op) {
- MaybePoisonOperand = Op;
+ bool HadMaybePoisonOperands = !MaybePoisonOperands.empty();
+ bool IsNewMaybePoisonOperand = MaybePoisonOperands.insert(Op);
+ if (!HadMaybePoisonOperands)
continue;
+ if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) {
+ // Multiple maybe-poison ops when not allowed - bail out.
+ return SDValue();
}
- // Multiple maybe-poison ops - bail out.
- return SDValue();
}
- assert(MaybePoisonOperand && "Should have found maybe-poison operand.");
+ assert(!MaybePoisonOperands.empty() &&
+ "Should have found maybe-poison operands.");
+
+ for (SDValue MaybePoisonOperand : MaybePoisonOperands) {
+ // First, freeze each offending operand.
+ SDValue FrozenMaybePoisonOperand = DAG.getFreeze(MaybePoisonOperand);
+ // Then, change all other uses of unfrozen operand to use frozen operand.
+ DAG.ReplaceAllUsesOfValueWith(MaybePoisonOperand, FrozenMaybePoisonOperand);
+ // But, that also updated the use in the freeze we just created, thus
+ // creating a cycle in a DAG. Let's undo that by mutating the freeze.
+ DAG.UpdateNodeOperands(FrozenMaybePoisonOperand.getNode(),
+ MaybePoisonOperand);
+ }
- // First, freeze the offending operand.
- SDValue FrozenMaybePoisonOperand = DAG.getFreeze(MaybePoisonOperand);
- // Then, change all other uses of unfrozen operand to use frozen operand.
- DAG.ReplaceAllUsesOfValueWith(MaybePoisonOperand, FrozenMaybePoisonOperand);
- // But, that also updated the use in the freeze we just created, thus
- // creating a cycle in a DAG. Let's undo that by mutating the freeze.
- DAG.UpdateNodeOperands(FrozenMaybePoisonOperand.getNode(),
- MaybePoisonOperand);
- // Finally, recreate the node with the frozen maybe-poison operand.
+ // Finally, recreate the node, it's operands were updated to use
+ // frozen operands, so we just need to use it's "original" operands.
SmallVector<SDValue> Ops(N0->op_begin(), N0->op_end());
// TODO: Just strip poison generating flags?
SDValue R = DAG.getNode(N0.getOpcode(), SDLoc(N0), N0->getVTList(), Ops);
diff --git a/llvm/test/CodeGen/X86/freeze-vector.ll b/llvm/test/CodeGen/X86/freeze-vector.ll
index 4604416e98e4..a074a77b3214 100644
--- a/llvm/test/CodeGen/X86/freeze-vector.ll
+++ b/llvm/test/CodeGen/X86/freeze-vector.ll
@@ -337,13 +337,17 @@ define void @freeze_two_frozen_buildvectors(ptr %origin0, ptr %origin1, ptr %dst
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl (%edx), %edx
; X86-NEXT: andl $15, %edx
-; X86-NEXT: vmovdqa {{.*#+}} xmm0 = [7,7,7,7]
-; X86-NEXT: vmovd %edx, %xmm1
-; X86-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]
-; X86-NEXT: vpand %xmm0, %xmm2, %xmm2
-; X86-NEXT: vmovdqa %xmm2, (%ecx)
-; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; X86-NEXT: vpand %xmm0, %xmm1, %xmm0
+; X86-NEXT: vmovd %eax, %xmm0
+; X86-NEXT: vpinsrd $1, %edx, %xmm0, %xmm1
+; X86-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
+; X86-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1
+; X86-NEXT: vmovdqa {{.*#+}} xmm2 = [7,7,7,7]
+; X86-NEXT: vpand %xmm2, %xmm1, %xmm1
+; X86-NEXT: vmovdqa %xmm1, (%ecx)
+; X86-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; X86-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; X86-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; X86-NEXT: vpand %xmm2, %xmm0, %xmm0
; X86-NEXT: vmovdqa %xmm0, (%eax)
; X86-NEXT: retl
;
@@ -351,11 +355,17 @@ define void @freeze_two_frozen_buildvectors(ptr %origin0, ptr %origin1, ptr %dst
; X64: # %bb.0:
; X64-NEXT: movl (%rdi), %eax
; X64-NEXT: andl $15, %eax
-; X64-NEXT: vpbroadcastd {{.*#+}} xmm0 = [7,7,7,7]
-; X64-NEXT: vmovd %eax, %xmm1
-; X64-NEXT: vpbroadcastd %xmm1, %xmm1
-; X64-NEXT: vpand %xmm0, %xmm1, %xmm0
-; X64-NEXT: vmovdqa %xmm0, (%rdx)
+; X64-NEXT: vmovd %eax, %xmm0
+; X64-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1
+; X64-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
+; X64-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1
+; X64-NEXT: vpbroadcastd {{.*#+}} xmm2 = [7,7,7,7]
+; X64-NEXT: vpand %xmm2, %xmm1, %xmm1
+; X64-NEXT: vmovdqa %xmm1, (%rdx)
+; X64-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; X64-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0
+; X64-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; X64-NEXT: vpand %xmm2, %xmm0, %xmm0
; X64-NEXT: vmovdqa %xmm0, (%rcx)
; X64-NEXT: retq
%i0.src = load i32, ptr %origin0
@@ -381,13 +391,17 @@ define void @freeze_two_buildvectors_only_one_frozen(ptr %origin0, ptr %origin1,
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl (%edx), %edx
; X86-NEXT: andl $15, %edx
-; X86-NEXT: vmovdqa {{.*#+}} xmm0 = [7,7,7,7]
-; X86-NEXT: vmovd %edx, %xmm1
-; X86-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[0,0,1,1]
-; X86-NEXT: vpand %xmm0, %xmm2, %xmm2
-; X86-NEXT: vmovdqa %xmm2, (%ecx)
-; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; X86-NEXT: vpand %xmm0, %xmm1, %xmm0
+; X86-NEXT: vmovd %eax, %xmm0
+; X86-NEXT: vpinsrd $1, %edx, %xmm0, %xmm1
+; X86-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
+; X86-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1
+; X86-NEXT: vmovdqa {{.*#+}} xmm2 = [7,7,7,7]
+; X86-NEXT: vpand %xmm2, %xmm1, %xmm1
+; X86-NEXT: vmovdqa %xmm1, (%ecx)
+; X86-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; X86-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; X86-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; X86-NEXT: vpand %xmm2, %xmm0, %xmm0
; X86-NEXT: vmovdqa %xmm0, (%eax)
; X86-NEXT: retl
;
@@ -396,10 +410,16 @@ define void @freeze_two_buildvectors_only_one_frozen(ptr %origin0, ptr %origin1,
; X64-NEXT: movl (%rdi), %eax
; X64-NEXT: andl $15, %eax
; X64-NEXT: vmovd %eax, %xmm0
-; X64-NEXT: vpbroadcastd %xmm0, %xmm0
-; X64-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,7,7,7]
-; X64-NEXT: vpand %xmm1, %xmm0, %xmm0
-; X64-NEXT: vmovdqa %xmm0, (%rdx)
+; X64-NEXT: vpinsrd $1, %eax, %xmm0, %xmm1
+; X64-NEXT: vpinsrd $2, %eax, %xmm1, %xmm1
+; X64-NEXT: vpinsrd $3, %eax, %xmm1, %xmm1
+; X64-NEXT: vpbroadcastd {{.*#+}} xmm2 = [7,7,7,7]
+; X64-NEXT: vpand %xmm2, %xmm1, %xmm1
+; X64-NEXT: vmovdqa %xmm1, (%rdx)
+; X64-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; X64-NEXT: vpinsrd $2, %eax, %xmm0, %xmm0
+; X64-NEXT: vpinsrd $3, %eax, %xmm0, %xmm0
+; X64-NEXT: vpand %xmm2, %xmm0, %xmm0
; X64-NEXT: vmovdqa %xmm0, (%rcx)
; X64-NEXT: retq
%i0.src = load i32, ptr %origin0
@@ -431,7 +451,6 @@ define void @freeze_buildvector(ptr %origin0, ptr %origin1, ptr %origin2, ptr %o
; X86-NEXT: vpinsrd $2, (%edx), %xmm0, %xmm0
; X86-NEXT: vpinsrd $3, (%ecx), %xmm0, %xmm0
; X86-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
; X86-NEXT: vmovdqa %xmm0, (%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -443,8 +462,6 @@ define void @freeze_buildvector(ptr %origin0, ptr %origin1, ptr %origin2, ptr %o
; X64-NEXT: vpinsrd $1, (%rsi), %xmm0, %xmm0
; X64-NEXT: vpinsrd $2, (%rdx), %xmm0, %xmm0
; X64-NEXT: vpinsrd $3, (%rcx), %xmm0, %xmm0
-; X64-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15]
-; X64-NEXT: vpand %xmm1, %xmm0, %xmm0
; X64-NEXT: vpbroadcastd {{.*#+}} xmm1 = [7,7,7,7]
; X64-NEXT: vpand %xmm1, %xmm0, %xmm0
; X64-NEXT: vmovdqa %xmm0, (%r8)
More information about the llvm-commits
mailing list