[llvm] [SelectionDAG][RISCV] Avoid store merging across function calls (PR #130430)
Mikhail R. Gadelha via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 21 08:47:59 PDT 2025
https://github.com/mikhailramalho updated https://github.com/llvm/llvm-project/pull/130430
>From 3e7e7081b9cd03a9f2f1191b08973b9602bf30f8 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Fri, 28 Feb 2025 13:23:43 -0300
Subject: [PATCH 01/29] [SelectionDAG] Avoid store merging across function
calls
This patch improves DAGCombiner's handling of potential store merges by
detecting function calls between loads and stores. When a function call
exists in the chain between a load and its corresponding store, we avoid
merging these stores as it would require costly register spilling.
Currently it's only enabled for riscv.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 4 ++
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 44 +++++++++++++++++--
llvm/lib/Target/RISCV/RISCVISelLowering.h | 6 +++
3 files changed, 50 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 2089d47e9cbc8..5e61c1f1a9687 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3506,6 +3506,10 @@ class TargetLoweringBase {
/// The default implementation just freezes the set of reserved registers.
virtual void finalizeLowering(MachineFunction &MF) const;
+ /// Returns true if it's profitable to allow merging store of loads when there
+ /// are functions calls between the load and the store.
+ virtual bool shouldMergeStoreOfLoadsOverCall() const { return true; }
+
//===----------------------------------------------------------------------===//
// GlobalISel Hooks
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index ef5f2210573e0..42d972b3a1db1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21363,8 +21363,38 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
// must not be zext, volatile, indexed, and they must be consecutive.
BaseIndexOffset LdBasePtr;
- for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
- StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
+ // Check if a call exists in the store chain.
+ auto HasCallInLdStChain = [](SDNode *Load, SDNode *Store) {
+ bool FoundCall = false;
+ SmallVector<SDNode *, 8> Nodes = {Store->getOperand(0).getNode()};
+ while (!Nodes.empty()) {
+ SDNode *Node = Nodes.pop_back_val();
+ if (Node->getNumOperands() == 0)
+ continue;
+
+ switch (Node->getOpcode()) {
+ case ISD::TokenFactor:
+ for (unsigned Nops = Node->getNumOperands(); Nops;)
+ Nodes.push_back(Node->getOperand(--Nops).getNode());
+ break;
+ case ISD::CALLSEQ_START:
+ FoundCall = true;
+ break;
+ case ISD::LOAD:
+ if (Node == Load)
+ return false;
+ [[fallthrough]];
+ default:
+ Nodes.push_back(Node->getOperand(0).getNode());
+ break;
+ }
+ }
+ return FoundCall;
+ };
+
+ auto StIt = StoreNodes.begin();
+ while (StIt != StoreNodes.end()) {
+ StoreSDNode *St = cast<StoreSDNode>(StIt->MemNode);
SDValue Val = peekThroughBitcasts(St->getValue());
LoadSDNode *Ld = cast<LoadSDNode>(Val);
@@ -21380,8 +21410,14 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
LdBasePtr = LdPtr;
}
- // We found a potential memory operand to merge.
- LoadNodes.push_back(MemOpLink(Ld, LdOffset));
+ // Check if there is a call in the load/store chain.
+ if (!TLI.shouldMergeStoreOfLoadsOverCall() &&
+ HasCallInLdStChain(Ld, St)) {
+ StIt = StoreNodes.erase(StIt);
+ } else {
+ LoadNodes.push_back(MemOpLink(Ld, LdOffset));
+ ++StIt;
+ }
}
while (NumConsecutiveStores >= 2 && LoadNodes.size() >= 2) {
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index ffbc14a29006c..d52d92eb581ee 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -1070,6 +1070,12 @@ class RISCVTargetLowering : public TargetLowering {
return false;
}
+ /// Disables storing and loading vectors when there are function calls between
+ /// the load and store, since these are more expensive than just using scalars
+ bool shouldMergeStoreOfLoadsOverCall() const override {
+ return false;
+ }
+
/// For available scheduling models FDIV + two independent FMULs are much
/// faster than two FDIVs.
unsigned combineRepeatedFPDivisors() const override;
>From 5b1fc65565deb0d655005bfca17b68e20a38cb05 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Mon, 3 Mar 2025 21:11:15 -0300
Subject: [PATCH 02/29] Check call_end instead of start
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 42d972b3a1db1..434a7f92c93b5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21377,7 +21377,7 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
for (unsigned Nops = Node->getNumOperands(); Nops;)
Nodes.push_back(Node->getOperand(--Nops).getNode());
break;
- case ISD::CALLSEQ_START:
+ case ISD::CALLSEQ_END:
FoundCall = true;
break;
case ISD::LOAD:
>From be40ec2d536dd5750f0adf2288c9b6d5d6e15e5c Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Mon, 3 Mar 2025 21:11:33 -0300
Subject: [PATCH 03/29] Only walk over mem operation chains
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 434a7f92c93b5..4374811fe4930 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21384,8 +21384,9 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
if (Node == Load)
return false;
[[fallthrough]];
- default:
+ case ISD::STORE:
Nodes.push_back(Node->getOperand(0).getNode());
+ default:
break;
}
}
>From c3298ffa4b8c1702d9dc4f32c9ac4ac3d1e24188 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Mon, 3 Mar 2025 21:12:14 -0300
Subject: [PATCH 04/29] Don't go over NumConsecutiveStores
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 4374811fe4930..75ce5b9cd2595 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21394,7 +21394,8 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
};
auto StIt = StoreNodes.begin();
- while (StIt != StoreNodes.end()) {
+ unsigned i = 0;
+ while (StIt != StoreNodes.end() && i < NumConsecutiveStores) {
StoreSDNode *St = cast<StoreSDNode>(StIt->MemNode);
SDValue Val = peekThroughBitcasts(St->getValue());
LoadSDNode *Ld = cast<LoadSDNode>(Val);
@@ -21412,13 +21413,13 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
}
// Check if there is a call in the load/store chain.
- if (!TLI.shouldMergeStoreOfLoadsOverCall() &&
- HasCallInLdStChain(Ld, St)) {
+ if (!TLI.shouldMergeStoreOfLoadsOverCall() && HasCallInLdStChain(Ld, St)) {
StIt = StoreNodes.erase(StIt);
} else {
LoadNodes.push_back(MemOpLink(Ld, LdOffset));
++StIt;
}
+ ++i;
}
while (NumConsecutiveStores >= 2 && LoadNodes.size() >= 2) {
>From b56d1b130bfa3f5533fe72b901ed8fb011f3459d Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Tue, 4 Mar 2025 14:09:23 -0300
Subject: [PATCH 05/29] Use SDValues
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 75ce5b9cd2595..bb987d5f8811a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21366,16 +21366,15 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
// Check if a call exists in the store chain.
auto HasCallInLdStChain = [](SDNode *Load, SDNode *Store) {
bool FoundCall = false;
- SmallVector<SDNode *, 8> Nodes = {Store->getOperand(0).getNode()};
- while (!Nodes.empty()) {
- SDNode *Node = Nodes.pop_back_val();
+ SmallVector<SDValue, 8> Values = {Store->getOperand(0)};
+ while (!Values.empty()) {
+ SDNode *Node = Values.pop_back_val().getNode();
if (Node->getNumOperands() == 0)
continue;
switch (Node->getOpcode()) {
case ISD::TokenFactor:
- for (unsigned Nops = Node->getNumOperands(); Nops;)
- Nodes.push_back(Node->getOperand(--Nops).getNode());
+ append_range(Values, Node->op_values());
break;
case ISD::CALLSEQ_END:
FoundCall = true;
@@ -21385,7 +21384,7 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
return false;
[[fallthrough]];
case ISD::STORE:
- Nodes.push_back(Node->getOperand(0).getNode());
+ Values.push_back(Node->getOperand(0));
default:
break;
}
>From 82420c71c3d4a279376faf752c4554204c68020e Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Tue, 4 Mar 2025 14:09:36 -0300
Subject: [PATCH 06/29] Added fallthrough
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index bb987d5f8811a..6209a3e8aa4a7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21385,6 +21385,7 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
[[fallthrough]];
case ISD::STORE:
Values.push_back(Node->getOperand(0));
+ [[fallthrough]];
default:
break;
}
>From 96c8e5366ec2defe12c0a1e16c3bc2e7ab256079 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Tue, 4 Mar 2025 21:58:18 -0300
Subject: [PATCH 07/29] Add Visited list to cache the walk
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 6209a3e8aa4a7..43c0d0f141989 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21365,16 +21365,20 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
// Check if a call exists in the store chain.
auto HasCallInLdStChain = [](SDNode *Load, SDNode *Store) {
+ SmallPtrSet<const SDNode *, 32> Visited;
+ SmallVector<const SDNode *, 8> Worklist;
+ Worklist.push_back(Store->getOperand(0).getNode());
+
bool FoundCall = false;
- SmallVector<SDValue, 8> Values = {Store->getOperand(0)};
- while (!Values.empty()) {
- SDNode *Node = Values.pop_back_val().getNode();
- if (Node->getNumOperands() == 0)
+ while (!Worklist.empty()) {
+ auto Node = Worklist.pop_back_val();
+ if (!Visited.insert(Node).second || Node->getNumOperands() == 0)
continue;
switch (Node->getOpcode()) {
case ISD::TokenFactor:
- append_range(Values, Node->op_values());
+ for (SDValue Op : Node->ops())
+ Worklist.push_back(Op.getNode());
break;
case ISD::CALLSEQ_END:
FoundCall = true;
@@ -21384,7 +21388,7 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
return false;
[[fallthrough]];
case ISD::STORE:
- Values.push_back(Node->getOperand(0));
+ Worklist.push_back(Node->getOperand(0).getNode());
[[fallthrough]];
default:
break;
>From 35743700d5aa4e514f55ba266deda8efe81fa885 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Tue, 4 Mar 2025 22:44:58 -0300
Subject: [PATCH 08/29] Moved increment
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 43c0d0f141989..f898e9e49c059 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21399,7 +21399,7 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
auto StIt = StoreNodes.begin();
unsigned i = 0;
- while (StIt != StoreNodes.end() && i < NumConsecutiveStores) {
+ while (StIt != StoreNodes.end() && i++ < NumConsecutiveStores) {
StoreSDNode *St = cast<StoreSDNode>(StIt->MemNode);
SDValue Val = peekThroughBitcasts(St->getValue());
LoadSDNode *Ld = cast<LoadSDNode>(Val);
@@ -21423,7 +21423,6 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
LoadNodes.push_back(MemOpLink(Ld, LdOffset));
++StIt;
}
- ++i;
}
while (NumConsecutiveStores >= 2 && LoadNodes.size() >= 2) {
>From f9393d5c208441c640389503f87b713a01437219 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Tue, 4 Mar 2025 23:19:38 -0300
Subject: [PATCH 09/29] Updated test case
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
.../CodeGen/RISCV/stores-of-loads-merging.ll | 24 +++++++++----------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
index b2be401b4676f..71bb4d5f41e7d 100644
--- a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
+++ b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
@@ -13,40 +13,40 @@ define void @f(ptr %m, ptr %n, ptr %p, ptr %q, ptr %r, ptr %s, double %t) {
; CHECK-NEXT: sd s0, 32(sp) # 8-byte Folded Spill
; CHECK-NEXT: sd s1, 24(sp) # 8-byte Folded Spill
; CHECK-NEXT: sd s2, 16(sp) # 8-byte Folded Spill
+; CHECK-NEXT: sd s3, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT: sd s4, 0(sp) # 8-byte Folded Spill
; CHECK-NEXT: .cfi_offset ra, -8
; CHECK-NEXT: .cfi_offset s0, -16
; CHECK-NEXT: .cfi_offset s1, -24
; CHECK-NEXT: .cfi_offset s2, -32
-; CHECK-NEXT: csrr a6, vlenb
-; CHECK-NEXT: sub sp, sp, a6
-; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x30, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 48 + 1 * vlenb
+; CHECK-NEXT: .cfi_offset s3, -40
+; CHECK-NEXT: .cfi_offset s4, -48
; CHECK-NEXT: mv s0, a5
; CHECK-NEXT: mv s1, a4
; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma
; CHECK-NEXT: vle64.v v8, (a0)
; CHECK-NEXT: vse64.v v8, (a1)
-; CHECK-NEXT: vle64.v v8, (a2)
-; CHECK-NEXT: addi a0, sp, 16
-; CHECK-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT: ld s3, 0(a2)
+; CHECK-NEXT: ld s4, 8(a2)
; CHECK-NEXT: mv s2, a3
; CHECK-NEXT: call g
-; CHECK-NEXT: addi a0, sp, 16
-; CHECK-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT: sd s3, 0(s2)
+; CHECK-NEXT: sd s4, 8(s2)
; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT: vse64.v v8, (s2)
; CHECK-NEXT: vle64.v v8, (s1)
; CHECK-NEXT: vse64.v v8, (s0)
-; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: add sp, sp, a0
-; CHECK-NEXT: .cfi_def_cfa sp, 48
; CHECK-NEXT: ld ra, 40(sp) # 8-byte Folded Reload
; CHECK-NEXT: ld s0, 32(sp) # 8-byte Folded Reload
; CHECK-NEXT: ld s1, 24(sp) # 8-byte Folded Reload
; CHECK-NEXT: ld s2, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT: ld s3, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT: ld s4, 0(sp) # 8-byte Folded Reload
; CHECK-NEXT: .cfi_restore ra
; CHECK-NEXT: .cfi_restore s0
; CHECK-NEXT: .cfi_restore s1
; CHECK-NEXT: .cfi_restore s2
+; CHECK-NEXT: .cfi_restore s3
+; CHECK-NEXT: .cfi_restore s4
; CHECK-NEXT: addi sp, sp, 48
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: ret
>From d86ec01d8494974d3d41e48a9081e1e8bc158b0a Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Wed, 5 Mar 2025 14:31:53 -0300
Subject: [PATCH 10/29] Enable merge by default for scalars
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/include/llvm/CodeGen/TargetLowering.h | 2 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 ++-
llvm/lib/Target/RISCV/RISCVISelLowering.h | 9 +++++----
3 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 5e61c1f1a9687..f8dd6cdd6aec8 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3508,7 +3508,7 @@ class TargetLoweringBase {
/// Returns true if it's profitable to allow merging store of loads when there
/// are functions calls between the load and the store.
- virtual bool shouldMergeStoreOfLoadsOverCall() const { return true; }
+ virtual bool shouldMergeStoreOfLoadsOverCall(EVT) const { return true; }
//===----------------------------------------------------------------------===//
// GlobalISel Hooks
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f898e9e49c059..11b6d516135ec 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21417,7 +21417,8 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
}
// Check if there is a call in the load/store chain.
- if (!TLI.shouldMergeStoreOfLoadsOverCall() && HasCallInLdStChain(Ld, St)) {
+ if (!TLI.shouldMergeStoreOfLoadsOverCall(MemVT) &&
+ HasCallInLdStChain(Ld, St)) {
StIt = StoreNodes.erase(StIt);
} else {
LoadNodes.push_back(MemOpLink(Ld, LdOffset));
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index d52d92eb581ee..658d1bce2cf6e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -1070,10 +1070,11 @@ class RISCVTargetLowering : public TargetLowering {
return false;
}
- /// Disables storing and loading vectors when there are function calls between
- /// the load and store, since these are more expensive than just using scalars
- bool shouldMergeStoreOfLoadsOverCall() const override {
- return false;
+ /// Disables storing and loading vectors by default when there are function
+ /// calls between the load and store, since these are more expensive than just
+ /// using scalars
+ bool shouldMergeStoreOfLoadsOverCall(EVT VT) const override {
+ return VT.isScalarInteger();
}
/// For available scheduling models FDIV + two independent FMULs are much
>From 04bca6d28a55df2da7714f7b2f8105e502d32baa Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Wed, 5 Mar 2025 14:32:54 -0300
Subject: [PATCH 11/29] Rewrite walk back algo to keep track of calls found
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 24 +++++++++----------
1 file changed, 11 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 11b6d516135ec..9ea06d70b36a0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21366,35 +21366,33 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
// Check if a call exists in the store chain.
auto HasCallInLdStChain = [](SDNode *Load, SDNode *Store) {
SmallPtrSet<const SDNode *, 32> Visited;
- SmallVector<const SDNode *, 8> Worklist;
- Worklist.push_back(Store->getOperand(0).getNode());
+ SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
+ Worklist.emplace_back(Store->getOperand(0).getNode(), false);
- bool FoundCall = false;
while (!Worklist.empty()) {
- auto Node = Worklist.pop_back_val();
+ auto [Node, FoundCall] = Worklist.pop_back_val();
if (!Visited.insert(Node).second || Node->getNumOperands() == 0)
continue;
switch (Node->getOpcode()) {
+ case ISD::CALLSEQ_END:
+ Worklist.emplace_back(Node->getOperand(0).getNode(), true);
+ break;
case ISD::TokenFactor:
for (SDValue Op : Node->ops())
- Worklist.push_back(Op.getNode());
- break;
- case ISD::CALLSEQ_END:
- FoundCall = true;
+ Worklist.emplace_back(Op.getNode(), FoundCall);
break;
case ISD::LOAD:
if (Node == Load)
- return false;
- [[fallthrough]];
- case ISD::STORE:
- Worklist.push_back(Node->getOperand(0).getNode());
+ return FoundCall;
[[fallthrough]];
default:
+ if (Node->getNumOperands() > 0)
+ Worklist.emplace_back(Node->getOperand(0).getNode(), FoundCall);
break;
}
}
- return FoundCall;
+ return false;
};
auto StIt = StoreNodes.begin();
>From f27092ff009d14ac87bde4118e91429fc009e6a6 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Thu, 6 Mar 2025 12:10:04 -0300
Subject: [PATCH 12/29] Check final type before we prevent merges
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 98 ++++++++++---------
1 file changed, 54 insertions(+), 44 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 9ea06d70b36a0..85b3682318e32 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21363,42 +21363,8 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
// must not be zext, volatile, indexed, and they must be consecutive.
BaseIndexOffset LdBasePtr;
- // Check if a call exists in the store chain.
- auto HasCallInLdStChain = [](SDNode *Load, SDNode *Store) {
- SmallPtrSet<const SDNode *, 32> Visited;
- SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
- Worklist.emplace_back(Store->getOperand(0).getNode(), false);
-
- while (!Worklist.empty()) {
- auto [Node, FoundCall] = Worklist.pop_back_val();
- if (!Visited.insert(Node).second || Node->getNumOperands() == 0)
- continue;
-
- switch (Node->getOpcode()) {
- case ISD::CALLSEQ_END:
- Worklist.emplace_back(Node->getOperand(0).getNode(), true);
- break;
- case ISD::TokenFactor:
- for (SDValue Op : Node->ops())
- Worklist.emplace_back(Op.getNode(), FoundCall);
- break;
- case ISD::LOAD:
- if (Node == Load)
- return FoundCall;
- [[fallthrough]];
- default:
- if (Node->getNumOperands() > 0)
- Worklist.emplace_back(Node->getOperand(0).getNode(), FoundCall);
- break;
- }
- }
- return false;
- };
-
- auto StIt = StoreNodes.begin();
- unsigned i = 0;
- while (StIt != StoreNodes.end() && i++ < NumConsecutiveStores) {
- StoreSDNode *St = cast<StoreSDNode>(StIt->MemNode);
+ for (unsigned i = 0; i < NumConsecutiveStores; ++i) {
+ StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
SDValue Val = peekThroughBitcasts(St->getValue());
LoadSDNode *Ld = cast<LoadSDNode>(Val);
@@ -21414,14 +21380,8 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
LdBasePtr = LdPtr;
}
- // Check if there is a call in the load/store chain.
- if (!TLI.shouldMergeStoreOfLoadsOverCall(MemVT) &&
- HasCallInLdStChain(Ld, St)) {
- StIt = StoreNodes.erase(StIt);
- } else {
- LoadNodes.push_back(MemOpLink(Ld, LdOffset));
- ++StIt;
- }
+ // We found a potential memory operand to merge.
+ LoadNodes.push_back(MemOpLink(Ld, LdOffset));
}
while (NumConsecutiveStores >= 2 && LoadNodes.size() >= 2) {
@@ -21593,6 +21553,56 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
JointMemOpVT = EVT::getIntegerVT(Context, SizeInBits);
}
+ auto HasCallInLdStChain = [](SmallVectorImpl<MemOpLink> &StoreNodes,
+ SmallVectorImpl<MemOpLink> &LoadNodes,
+ unsigned NumStores) {
+ for (unsigned i = 0; i < NumStores; ++i) {
+ StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
+ SDValue Val = peekThroughBitcasts(St->getValue());
+ LoadSDNode *Ld = cast<LoadSDNode>(Val);
+ assert(Ld == LoadNodes[i].MemNode && "Load and store mismatch");
+
+ SmallPtrSet<const SDNode *, 32> Visited;
+ SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
+ Worklist.emplace_back(St->getOperand(0).getNode(), false);
+
+ while (!Worklist.empty()) {
+ auto [Node, FoundCall] = Worklist.pop_back_val();
+ if (!Visited.insert(Node).second || Node->getNumOperands() == 0)
+ continue;
+
+ switch (Node->getOpcode()) {
+ case ISD::CALLSEQ_END:
+ Worklist.emplace_back(Node->getOperand(0).getNode(), true);
+ break;
+ case ISD::TokenFactor:
+ for (SDValue Op : Node->ops())
+ Worklist.emplace_back(Op.getNode(), FoundCall);
+ break;
+ case ISD::LOAD:
+ if (Node == Ld)
+ return FoundCall;
+ [[fallthrough]];
+ default:
+ if (Node->getNumOperands() > 0)
+ Worklist.emplace_back(Node->getOperand(0).getNode(), FoundCall);
+ break;
+ }
+ }
+ return false;
+ }
+ return false;
+ };
+
+ // Check if there is a call in the load/store chain.
+ if (!TLI.shouldMergeStoreOfLoadsOverCall(JointMemOpVT) &&
+ HasCallInLdStChain(StoreNodes, LoadNodes, NumElem)) {
+ StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
+ LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + NumElem);
+ NumConsecutiveStores -= NumElem;
+ continue;
+ }
+
SDLoc LoadDL(LoadNodes[0].MemNode);
SDLoc StoreDL(StoreNodes[0].MemNode);
>From 9faa629fd23dcf359d6d9d66d8e68f9d90c461fe Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Mon, 17 Mar 2025 11:43:52 -0300
Subject: [PATCH 13/29] No need to check operands. It's checked in the start of
the loop
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 85b3682318e32..7804b74579eba 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21584,8 +21584,7 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
return FoundCall;
[[fallthrough]];
default:
- if (Node->getNumOperands() > 0)
- Worklist.emplace_back(Node->getOperand(0).getNode(), FoundCall);
+ Worklist.emplace_back(Node->getOperand(0).getNode(), FoundCall);
break;
}
}
>From b326da182c749a20b9f3fce971bb7951d00d31a2 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Mon, 17 Mar 2025 11:46:54 -0300
Subject: [PATCH 14/29] Assert operand type
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 7804b74579eba..2618781c7abb4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21584,6 +21584,8 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
return FoundCall;
[[fallthrough]];
default:
+ assert(Node->getOperand(0).getValueType() == MVT::Other &&
+ "Invalid chain type");
Worklist.emplace_back(Node->getOperand(0).getNode(), FoundCall);
break;
}
>From c8580206962240caa157bfe6c8f48b3aa360ea22 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Mon, 17 Mar 2025 12:26:50 -0300
Subject: [PATCH 15/29] Moved peekThroughBitcasts into an assertion
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 2618781c7abb4..04a860fb6fade 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21558,9 +21558,9 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
unsigned NumStores) {
for (unsigned i = 0; i < NumStores; ++i) {
StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
- SDValue Val = peekThroughBitcasts(St->getValue());
- LoadSDNode *Ld = cast<LoadSDNode>(Val);
- assert(Ld == LoadNodes[i].MemNode && "Load and store mismatch");
+ LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
+ assert(Ld == cast<LoadSDNode>(peekThroughBitcasts(St->getValue())) &&
+ "Load and store mismatch");
SmallPtrSet<const SDNode *, 32> Visited;
SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
>From b6b15211ebdca0c57f03dd74ae61ad4b5b975f06 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Mon, 17 Mar 2025 14:17:22 -0300
Subject: [PATCH 16/29] Use getChain instead of accessing the operand 0
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 04a860fb6fade..cc4ed68dc0da2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21564,7 +21564,7 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
SmallPtrSet<const SDNode *, 32> Visited;
SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
- Worklist.emplace_back(St->getOperand(0).getNode(), false);
+ Worklist.emplace_back(St->getChain().getNode(), false);
while (!Worklist.empty()) {
auto [Node, FoundCall] = Worklist.pop_back_val();
>From 18e68eaf897a717d37cc10e61f91d14bcf61593f Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Mon, 17 Mar 2025 22:42:49 -0300
Subject: [PATCH 17/29] Make hasCallInLdStChain a member function
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 92 ++++++++++---------
1 file changed, 49 insertions(+), 43 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index cc4ed68dc0da2..bbf04eca3eff3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -792,6 +792,12 @@ namespace {
SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
SDNode *RootNode);
+ /// Helper function for tryStoreMergeOfLoads. Checks if the load/store
+ /// chain has a call in it. \return True if a call is found.
+ bool hasCallInLdStChain(SmallVectorImpl<MemOpLink> &StoreNodes,
+ SmallVectorImpl<MemOpLink> &LoadNodes,
+ unsigned NumStores);
+
/// This is a helper function for mergeConsecutiveStores. Given a list of
/// store candidates, find the first N that are consecutive in memory.
/// Returns 0 if there are not at least 2 consecutive stores to try merging.
@@ -21107,6 +21113,48 @@ bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
return true;
}
+bool DAGCombiner::hasCallInLdStChain(SmallVectorImpl<MemOpLink> &StoreNodes,
+ SmallVectorImpl<MemOpLink> &LoadNodes,
+ unsigned NumStores) {
+ for (unsigned i = 0; i < NumStores; ++i) {
+ StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
+ LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
+ assert(Ld == cast<LoadSDNode>(peekThroughBitcasts(St->getValue())) &&
+ "Load and store mismatch");
+
+ SmallPtrSet<const SDNode *, 32> Visited;
+ SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
+ Worklist.emplace_back(St->getChain().getNode(), false);
+
+ while (!Worklist.empty()) {
+ auto [Node, FoundCall] = Worklist.pop_back_val();
+ if (!Visited.insert(Node).second || Node->getNumOperands() == 0)
+ continue;
+
+ switch (Node->getOpcode()) {
+ case ISD::CALLSEQ_END:
+ Worklist.emplace_back(Node->getOperand(0).getNode(), true);
+ break;
+ case ISD::TokenFactor:
+ for (SDValue Op : Node->ops())
+ Worklist.emplace_back(Op.getNode(), FoundCall);
+ break;
+ case ISD::LOAD:
+ if (Node == Ld)
+ return FoundCall;
+ [[fallthrough]];
+ default:
+ assert(Node->getOperand(0).getValueType() == MVT::Other &&
+ "Invalid chain type");
+ Worklist.emplace_back(Node->getOperand(0).getNode(), FoundCall);
+ break;
+ }
+ }
+ return false;
+ }
+ return false;
+}
+
unsigned
DAGCombiner::getConsecutiveStores(SmallVectorImpl<MemOpLink> &StoreNodes,
int64_t ElementSizeBytes) const {
@@ -21553,51 +21601,9 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
JointMemOpVT = EVT::getIntegerVT(Context, SizeInBits);
}
- auto HasCallInLdStChain = [](SmallVectorImpl<MemOpLink> &StoreNodes,
- SmallVectorImpl<MemOpLink> &LoadNodes,
- unsigned NumStores) {
- for (unsigned i = 0; i < NumStores; ++i) {
- StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
- LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
- assert(Ld == cast<LoadSDNode>(peekThroughBitcasts(St->getValue())) &&
- "Load and store mismatch");
-
- SmallPtrSet<const SDNode *, 32> Visited;
- SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
- Worklist.emplace_back(St->getChain().getNode(), false);
-
- while (!Worklist.empty()) {
- auto [Node, FoundCall] = Worklist.pop_back_val();
- if (!Visited.insert(Node).second || Node->getNumOperands() == 0)
- continue;
-
- switch (Node->getOpcode()) {
- case ISD::CALLSEQ_END:
- Worklist.emplace_back(Node->getOperand(0).getNode(), true);
- break;
- case ISD::TokenFactor:
- for (SDValue Op : Node->ops())
- Worklist.emplace_back(Op.getNode(), FoundCall);
- break;
- case ISD::LOAD:
- if (Node == Ld)
- return FoundCall;
- [[fallthrough]];
- default:
- assert(Node->getOperand(0).getValueType() == MVT::Other &&
- "Invalid chain type");
- Worklist.emplace_back(Node->getOperand(0).getNode(), FoundCall);
- break;
- }
- }
- return false;
- }
- return false;
- };
-
// Check if there is a call in the load/store chain.
if (!TLI.shouldMergeStoreOfLoadsOverCall(JointMemOpVT) &&
- HasCallInLdStChain(StoreNodes, LoadNodes, NumElem)) {
+ hasCallInLdStChain(StoreNodes, LoadNodes, NumElem)) {
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + NumElem);
NumConsecutiveStores -= NumElem;
>From 3bc2b224b1aa230adf0217bc1041c8fe6da3abf2 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Tue, 18 Mar 2025 15:37:17 -0300
Subject: [PATCH 18/29] Added test case
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
.../CodeGen/RISCV/stores-of-loads-merging.ll | 35 +++++++++++++++++++
1 file changed, 35 insertions(+)
diff --git a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
index 71bb4d5f41e7d..d697f911165b9 100644
--- a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
+++ b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
@@ -90,6 +90,41 @@ define void @f1(ptr %m, ptr %n, ptr %p, ptr %q, ptr %r, ptr %s, double %t) {
store i64 %x0, ptr %q
%q.1 = getelementptr i64, ptr %q, i64 1
store i64 %x1, ptr %q.1
+ ret void
+}
+define void @i8_i16(ptr %p, ptr %q) {
+; CHECK-LABEL: i8_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -32
+; CHECK-NEXT: .cfi_def_cfa_offset 32
+; CHECK-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; CHECK-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; CHECK-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT: .cfi_offset ra, -8
+; CHECK-NEXT: .cfi_offset s0, -16
+; CHECK-NEXT: .cfi_offset s1, -24
+; CHECK-NEXT: lh s1, 0(a0)
+; CHECK-NEXT: mv s0, a1
+; CHECK-NEXT: call g
+; CHECK-NEXT: sh s1, 0(s0)
+; CHECK-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; CHECK-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT: .cfi_restore ra
+; CHECK-NEXT: .cfi_restore s0
+; CHECK-NEXT: .cfi_restore s1
+; CHECK-NEXT: addi sp, sp, 32
+; CHECK-NEXT: .cfi_def_cfa_offset 0
+; CHECK-NEXT: ret
+ %p0 = getelementptr i8, ptr %p, i64 0
+ %p1 = getelementptr i8, ptr %p, i64 1
+ %x0 = load i8, ptr %p0, align 2
+ %x1 = load i8, ptr %p1
+ call void @g()
+ %q0 = getelementptr i8, ptr %q, i64 0
+ %q1 = getelementptr i8, ptr %q, i64 1
+ store i8 %x0, ptr %q0, align 2
+ store i8 %x1, ptr %q1
ret void
}
>From 904641ffb553145c38c0051be8a6c8c7812d0f26 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Wed, 19 Mar 2025 13:06:37 -0300
Subject: [PATCH 19/29] Removed duplicated test after merge
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
.../CodeGen/RISCV/stores-of-loads-merging.ll | 36 -------------------
1 file changed, 36 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
index 031d1d74cb9e7..d722dadc71565 100644
--- a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
+++ b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
@@ -95,42 +95,6 @@ define void @f1(ptr %m, ptr %n, ptr %p, ptr %q, ptr %r, ptr %s, double %t) {
ret void
}
-define void @i8_i16(ptr %p, ptr %q) {
-; CHECK-LABEL: i8_i16:
-; CHECK: # %bb.0:
-; CHECK-NEXT: addi sp, sp, -32
-; CHECK-NEXT: .cfi_def_cfa_offset 32
-; CHECK-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
-; CHECK-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
-; CHECK-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
-; CHECK-NEXT: .cfi_offset ra, -8
-; CHECK-NEXT: .cfi_offset s0, -16
-; CHECK-NEXT: .cfi_offset s1, -24
-; CHECK-NEXT: lh s1, 0(a0)
-; CHECK-NEXT: mv s0, a1
-; CHECK-NEXT: call g
-; CHECK-NEXT: sh s1, 0(s0)
-; CHECK-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
-; CHECK-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
-; CHECK-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
-; CHECK-NEXT: .cfi_restore ra
-; CHECK-NEXT: .cfi_restore s0
-; CHECK-NEXT: .cfi_restore s1
-; CHECK-NEXT: addi sp, sp, 32
-; CHECK-NEXT: .cfi_def_cfa_offset 0
-; CHECK-NEXT: ret
- %p0 = getelementptr i8, ptr %p, i64 0
- %p1 = getelementptr i8, ptr %p, i64 1
- %x0 = load i8, ptr %p0, align 2
- %x1 = load i8, ptr %p1
- call void @g()
- %q0 = getelementptr i8, ptr %q, i64 0
- %q1 = getelementptr i8, ptr %q, i64 1
- store i8 %x0, ptr %q0, align 2
- store i8 %x1, ptr %q1
- ret void
-}
-
; Merging scalars is profitable, it reduces pressure within a single
; register class.
define void @i8_i16(ptr %p, ptr %q) {
>From a255f16764cf8447e2c30d5c0579690788617951 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Wed, 19 Mar 2025 13:12:37 -0300
Subject: [PATCH 20/29] No need to declare intrinsics anymore
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
index d722dadc71565..95303f63f0776 100644
--- a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
+++ b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
@@ -1,7 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=riscv64 -mattr=+v | FileCheck %s
-declare i32 @llvm.experimental.constrained.fptosi.i32.f64(double, metadata)
declare void @g()
; TODO: Merging scalars into vectors is unprofitable because we have no
>From 75f4caa77cbea0cde909f77345c42d371edb8fb8 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Wed, 19 Mar 2025 13:15:59 -0300
Subject: [PATCH 21/29] Removed unused args
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
index 95303f63f0776..15f93954f559e 100644
--- a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
+++ b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
@@ -76,13 +76,13 @@ define void @f(ptr %m, ptr %n, ptr %p, ptr %q, ptr %r, ptr %s, double %t) {
ret void
}
-define void @f1(ptr %m, ptr %n, ptr %p, ptr %q, ptr %r, ptr %s, double %t) {
+define void @f1(ptr %p, ptr %q, double %t) {
; CHECK-LABEL: f1:
; CHECK: # %bb.0:
; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT: vle64.v v8, (a2)
+; CHECK-NEXT: vle64.v v8, (a0)
; CHECK-NEXT: fcvt.wu.d a0, fa0, rtz
-; CHECK-NEXT: vse64.v v8, (a3)
+; CHECK-NEXT: vse64.v v8, (a1)
; CHECK-NEXT: ret
%x0 = load i64, ptr %p
%p.1 = getelementptr i64, ptr %p, i64 1
>From de96633b1f048cf0c0737e1de924dc531a4a2625 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Wed, 19 Mar 2025 13:29:23 -0300
Subject: [PATCH 22/29] Address comment
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/include/llvm/CodeGen/TargetLowering.h | 2 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 2 +-
llvm/lib/Target/RISCV/RISCVISelLowering.h | 4 ++--
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 058d42ff962e5..58ac87206b9a6 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3522,7 +3522,7 @@ class TargetLoweringBase {
/// Returns true if it's profitable to allow merging store of loads when there
/// are functions calls between the load and the store.
- virtual bool shouldMergeStoreOfLoadsOverCall(EVT) const { return true; }
+ virtual bool shouldMergeStoreOfLoadsOverCall(EVT, EVT) const { return true; }
//===----------------------------------------------------------------------===//
// GlobalISel Hooks
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f62a930d39fd9..c5288147e8969 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21647,7 +21647,7 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
}
// Check if there is a call in the load/store chain.
- if (!TLI.shouldMergeStoreOfLoadsOverCall(JointMemOpVT) &&
+ if (!TLI.shouldMergeStoreOfLoadsOverCall(MemVT, JointMemOpVT) &&
hasCallInLdStChain(StoreNodes, LoadNodes, NumElem)) {
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + NumElem);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 658d1bce2cf6e..d52be42f8569c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -1073,8 +1073,8 @@ class RISCVTargetLowering : public TargetLowering {
/// Disables storing and loading vectors by default when there are function
/// calls between the load and store, since these are more expensive than just
/// using scalars
- bool shouldMergeStoreOfLoadsOverCall(EVT VT) const override {
- return VT.isScalarInteger();
+ bool shouldMergeStoreOfLoadsOverCall(EVT SrcVT, EVT MergedVT) const override {
+ return SrcVT.isScalarInteger() == MergedVT.isScalarInteger();
}
/// For available scheduling models FDIV + two independent FMULs are much
>From 69c361c1909ebbe9062109361746042b78381808 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Wed, 19 Mar 2025 14:58:25 -0300
Subject: [PATCH 23/29] Address comment
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index c5288147e8969..17520f80e43fe 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21195,7 +21195,6 @@ bool DAGCombiner::hasCallInLdStChain(SmallVectorImpl<MemOpLink> &StoreNodes,
break;
}
}
- return false;
}
return false;
}
>From 0dfd35496e4f93dac7b756142efdf705e6ade101 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Wed, 19 Mar 2025 15:03:07 -0300
Subject: [PATCH 24/29] Removed todo
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
index 15f93954f559e..f49c9d4f50f3b 100644
--- a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
+++ b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
@@ -3,8 +3,6 @@
declare void @g()
-; TODO: Merging scalars into vectors is unprofitable because we have no
-; vector CSRs which creates additional spills around the call.
define void @f(ptr %m, ptr %n, ptr %p, ptr %q, ptr %r, ptr %s, double %t) {
; CHECK-LABEL: f:
; CHECK: # %bb.0:
>From e73c49d37d92817cc1c8654a6587abece1aeb55a Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Wed, 19 Mar 2025 15:37:05 -0300
Subject: [PATCH 25/29] Simplify interface
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 67 +++++++++----------
1 file changed, 30 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 17520f80e43fe..af49a65e2f688 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -794,9 +794,7 @@ namespace {
/// Helper function for tryStoreMergeOfLoads. Checks if the load/store
/// chain has a call in it. \return True if a call is found.
- bool hasCallInLdStChain(SmallVectorImpl<MemOpLink> &StoreNodes,
- SmallVectorImpl<MemOpLink> &LoadNodes,
- unsigned NumStores);
+ bool hasCallInLdStChain(StoreSDNode *St, LoadSDNode *Ld);
/// This is a helper function for mergeConsecutiveStores. Given a list of
/// store candidates, find the first N that are consecutive in memory.
@@ -21158,42 +21156,36 @@ bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
return true;
}
-bool DAGCombiner::hasCallInLdStChain(SmallVectorImpl<MemOpLink> &StoreNodes,
- SmallVectorImpl<MemOpLink> &LoadNodes,
- unsigned NumStores) {
- for (unsigned i = 0; i < NumStores; ++i) {
- StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
- LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
- assert(Ld == cast<LoadSDNode>(peekThroughBitcasts(St->getValue())) &&
- "Load and store mismatch");
+bool DAGCombiner::hasCallInLdStChain(StoreSDNode *St, LoadSDNode *Ld) {
+ assert(Ld == cast<LoadSDNode>(peekThroughBitcasts(St->getValue())) &&
+ "Load and store mismatch");
- SmallPtrSet<const SDNode *, 32> Visited;
- SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
- Worklist.emplace_back(St->getChain().getNode(), false);
+ SmallPtrSet<const SDNode *, 32> Visited;
+ SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
+ Worklist.emplace_back(St->getChain().getNode(), false);
- while (!Worklist.empty()) {
- auto [Node, FoundCall] = Worklist.pop_back_val();
- if (!Visited.insert(Node).second || Node->getNumOperands() == 0)
- continue;
+ while (!Worklist.empty()) {
+ auto [Node, FoundCall] = Worklist.pop_back_val();
+ if (!Visited.insert(Node).second || Node->getNumOperands() == 0)
+ continue;
- switch (Node->getOpcode()) {
- case ISD::CALLSEQ_END:
- Worklist.emplace_back(Node->getOperand(0).getNode(), true);
- break;
- case ISD::TokenFactor:
- for (SDValue Op : Node->ops())
- Worklist.emplace_back(Op.getNode(), FoundCall);
- break;
- case ISD::LOAD:
- if (Node == Ld)
- return FoundCall;
- [[fallthrough]];
- default:
- assert(Node->getOperand(0).getValueType() == MVT::Other &&
- "Invalid chain type");
- Worklist.emplace_back(Node->getOperand(0).getNode(), FoundCall);
- break;
- }
+ switch (Node->getOpcode()) {
+ case ISD::CALLSEQ_END:
+ Worklist.emplace_back(Node->getOperand(0).getNode(), true);
+ break;
+ case ISD::TokenFactor:
+ for (SDValue Op : Node->ops())
+ Worklist.emplace_back(Op.getNode(), FoundCall);
+ break;
+ case ISD::LOAD:
+ if (Node == Ld)
+ return FoundCall;
+ [[fallthrough]];
+ default:
+ assert(Node->getOperand(0).getValueType() == MVT::Other &&
+ "Invalid chain type");
+ Worklist.emplace_back(Node->getOperand(0).getNode(), FoundCall);
+ break;
}
}
return false;
@@ -21647,7 +21639,8 @@ bool DAGCombiner::tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
// Check if there is a call in the load/store chain.
if (!TLI.shouldMergeStoreOfLoadsOverCall(MemVT, JointMemOpVT) &&
- hasCallInLdStChain(StoreNodes, LoadNodes, NumElem)) {
+ hasCallInLdStChain(cast<StoreSDNode>(StoreNodes[0].MemNode),
+ cast<LoadSDNode>(LoadNodes[0].MemNode))) {
StoreNodes.erase(StoreNodes.begin(), StoreNodes.begin() + NumElem);
LoadNodes.erase(LoadNodes.begin(), LoadNodes.begin() + NumElem);
NumConsecutiveStores -= NumElem;
>From d6c848db4a97103e19c398848bde0876a2c4c88e Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Thu, 20 Mar 2025 12:34:38 -0300
Subject: [PATCH 26/29] Remove assert that fails when building blender_r
This asserts fails because the order of the store/loads are reversed
prior to the check, if we can rotate the loads.
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 ---
1 file changed, 3 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index af49a65e2f688..cd2a7fc3f9ab6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -21157,9 +21157,6 @@ bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
}
bool DAGCombiner::hasCallInLdStChain(StoreSDNode *St, LoadSDNode *Ld) {
- assert(Ld == cast<LoadSDNode>(peekThroughBitcasts(St->getValue())) &&
- "Load and store mismatch");
-
SmallPtrSet<const SDNode *, 32> Visited;
SmallVector<std::pair<const SDNode *, bool>, 8> Worklist;
Worklist.emplace_back(St->getChain().getNode(), false);
>From 0189f30577ca94e460417a9c0d9e657215361035 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Thu, 20 Mar 2025 20:31:57 -0300
Subject: [PATCH 27/29] Address comment
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/lib/Target/RISCV/RISCVISelLowering.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index d52be42f8569c..4072f8ea2fc67 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -1074,7 +1074,7 @@ class RISCVTargetLowering : public TargetLowering {
/// calls between the load and store, since these are more expensive than just
/// using scalars
bool shouldMergeStoreOfLoadsOverCall(EVT SrcVT, EVT MergedVT) const override {
- return SrcVT.isScalarInteger() == MergedVT.isScalarInteger();
+ return !MergedVT.isVector() || SrcVT.isVector();
}
/// For available scheduling models FDIV + two independent FMULs are much
>From 67b3b65d9663ccebfdc5de9610ae59f796319176 Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Thu, 20 Mar 2025 21:05:25 -0300
Subject: [PATCH 28/29] Update test
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
.../CodeGen/RISCV/stores-of-loads-merging.ll | 78 +++++++++----------
1 file changed, 36 insertions(+), 42 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
index d38a634c7ac72..398b4844459ad 100644
--- a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
+++ b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
@@ -363,28 +363,26 @@ define void @two_half(ptr %p, ptr %q) {
; ZVFH-NEXT: .cfi_def_cfa_offset 32
; ZVFH-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; ZVFH-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; ZVFH-NEXT: fsd fs0, 8(sp) # 8-byte Folded Spill
+; ZVFH-NEXT: fsd fs1, 0(sp) # 8-byte Folded Spill
; ZVFH-NEXT: .cfi_offset ra, -8
; ZVFH-NEXT: .cfi_offset s0, -16
-; ZVFH-NEXT: csrr a2, vlenb
-; ZVFH-NEXT: sub sp, sp, a2
-; ZVFH-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 1 * vlenb
-; ZVFH-NEXT: vsetivli zero, 2, e16, mf4, ta, ma
-; ZVFH-NEXT: vle16.v v8, (a0)
-; ZVFH-NEXT: addi a0, sp, 16
-; ZVFH-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill
+; ZVFH-NEXT: .cfi_offset fs0, -24
+; ZVFH-NEXT: .cfi_offset fs1, -32
+; ZVFH-NEXT: flh fs0, 0(a0)
+; ZVFH-NEXT: flh fs1, 2(a0)
; ZVFH-NEXT: mv s0, a1
; ZVFH-NEXT: call g
-; ZVFH-NEXT: addi a0, sp, 16
-; ZVFH-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload
-; ZVFH-NEXT: vsetivli zero, 2, e16, mf4, ta, ma
-; ZVFH-NEXT: vse16.v v8, (s0)
-; ZVFH-NEXT: csrr a0, vlenb
-; ZVFH-NEXT: add sp, sp, a0
-; ZVFH-NEXT: .cfi_def_cfa sp, 32
+; ZVFH-NEXT: fsh fs0, 0(s0)
+; ZVFH-NEXT: fsh fs1, 2(s0)
; ZVFH-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
; ZVFH-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; ZVFH-NEXT: fld fs0, 8(sp) # 8-byte Folded Reload
+; ZVFH-NEXT: fld fs1, 0(sp) # 8-byte Folded Reload
; ZVFH-NEXT: .cfi_restore ra
; ZVFH-NEXT: .cfi_restore s0
+; ZVFH-NEXT: .cfi_restore fs0
+; ZVFH-NEXT: .cfi_restore fs1
; ZVFH-NEXT: addi sp, sp, 32
; ZVFH-NEXT: .cfi_def_cfa_offset 0
; ZVFH-NEXT: ret
@@ -409,28 +407,26 @@ define void @two_float(ptr %p, ptr %q) {
; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; CHECK-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; CHECK-NEXT: fsd fs0, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT: fsd fs1, 0(sp) # 8-byte Folded Spill
; CHECK-NEXT: .cfi_offset ra, -8
; CHECK-NEXT: .cfi_offset s0, -16
-; CHECK-NEXT: csrr a2, vlenb
-; CHECK-NEXT: sub sp, sp, a2
-; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 1 * vlenb
-; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT: vle32.v v8, (a0)
-; CHECK-NEXT: addi a0, sp, 16
-; CHECK-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT: .cfi_offset fs0, -24
+; CHECK-NEXT: .cfi_offset fs1, -32
+; CHECK-NEXT: flw fs0, 0(a0)
+; CHECK-NEXT: flw fs1, 4(a0)
; CHECK-NEXT: mv s0, a1
; CHECK-NEXT: call g
-; CHECK-NEXT: addi a0, sp, 16
-; CHECK-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT: vsetivli zero, 2, e32, mf2, ta, ma
-; CHECK-NEXT: vse32.v v8, (s0)
-; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: add sp, sp, a0
-; CHECK-NEXT: .cfi_def_cfa sp, 32
+; CHECK-NEXT: fsw fs0, 0(s0)
+; CHECK-NEXT: fsw fs1, 4(s0)
; CHECK-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
; CHECK-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT: fld fs0, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT: fld fs1, 0(sp) # 8-byte Folded Reload
; CHECK-NEXT: .cfi_restore ra
; CHECK-NEXT: .cfi_restore s0
+; CHECK-NEXT: .cfi_restore fs0
+; CHECK-NEXT: .cfi_restore fs1
; CHECK-NEXT: addi sp, sp, 32
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: ret
@@ -453,28 +449,26 @@ define void @two_double(ptr %p, ptr %q) {
; CHECK-NEXT: .cfi_def_cfa_offset 32
; CHECK-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
; CHECK-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; CHECK-NEXT: fsd fs0, 8(sp) # 8-byte Folded Spill
+; CHECK-NEXT: fsd fs1, 0(sp) # 8-byte Folded Spill
; CHECK-NEXT: .cfi_offset ra, -8
; CHECK-NEXT: .cfi_offset s0, -16
-; CHECK-NEXT: csrr a2, vlenb
-; CHECK-NEXT: sub sp, sp, a2
-; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x20, 0x22, 0x11, 0x01, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 32 + 1 * vlenb
-; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT: vle64.v v8, (a0)
-; CHECK-NEXT: addi a0, sp, 16
-; CHECK-NEXT: vs1r.v v8, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT: .cfi_offset fs0, -24
+; CHECK-NEXT: .cfi_offset fs1, -32
+; CHECK-NEXT: fld fs0, 0(a0)
+; CHECK-NEXT: fld fs1, 8(a0)
; CHECK-NEXT: mv s0, a1
; CHECK-NEXT: call g
-; CHECK-NEXT: addi a0, sp, 16
-; CHECK-NEXT: vl1r.v v8, (a0) # Unknown-size Folded Reload
-; CHECK-NEXT: vsetivli zero, 2, e64, m1, ta, ma
-; CHECK-NEXT: vse64.v v8, (s0)
-; CHECK-NEXT: csrr a0, vlenb
-; CHECK-NEXT: add sp, sp, a0
-; CHECK-NEXT: .cfi_def_cfa sp, 32
+; CHECK-NEXT: fsd fs0, 0(s0)
+; CHECK-NEXT: fsd fs1, 8(s0)
; CHECK-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
; CHECK-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; CHECK-NEXT: fld fs0, 8(sp) # 8-byte Folded Reload
+; CHECK-NEXT: fld fs1, 0(sp) # 8-byte Folded Reload
; CHECK-NEXT: .cfi_restore ra
; CHECK-NEXT: .cfi_restore s0
+; CHECK-NEXT: .cfi_restore fs0
+; CHECK-NEXT: .cfi_restore fs1
; CHECK-NEXT: addi sp, sp, 32
; CHECK-NEXT: .cfi_def_cfa_offset 0
; CHECK-NEXT: ret
>From ed8a5fd26ce91ad78fde2b745799878641994c9a Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail at igalia.com>
Date: Thu, 20 Mar 2025 21:33:13 -0300
Subject: [PATCH 29/29] Removed todo
Signed-off-by: Mikhail R. Gadelha <mikhail at igalia.com>
---
llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll | 2 --
1 file changed, 2 deletions(-)
diff --git a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
index 398b4844459ad..fefe72bd05f42 100644
--- a/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
+++ b/llvm/test/CodeGen/RISCV/stores-of-loads-merging.ll
@@ -398,8 +398,6 @@ define void @two_half(ptr %p, ptr %q) {
ret void
}
-; TODO: This one is currently a vector which is unprofitable, we should
-; use i64 instead.
define void @two_float(ptr %p, ptr %q) {
; CHECK-LABEL: two_float:
; CHECK: # %bb.0:
More information about the llvm-commits
mailing list