[llvm] [AArch64] Don't use LowerToPredicatedOp to shufflevector -> SVE lowerings (PR #140713)
Benjamin Maxwell via llvm-commits
llvm-commits at lists.llvm.org
Tue May 20 06:17:05 PDT 2025
https://github.com/MacDue updated https://github.com/llvm/llvm-project/pull/140713
>From fec92e49c04c88761156ebd44d9120a7e20f21c4 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 20 May 2025 11:14:09 +0000
Subject: [PATCH 1/2] [AArch64] Don't use LowerToPredicatedOp to shufflevector
-> SVE lowerings
The use of `LowerToPredicatedOp` here seems like a mistake as
`LowerToPredicatedOp` turns the SDValue passed to it into the desired
predicated node by copying over operands (and adding a predicate). This
results in two odd things here, the BITCASTs created and passed to
`LowerToPredicatedOp` are not used, only the operands of those bitcasts
are taken. Secondly, when a shuffle vector node is passed directly to
`LowerToPredicatedOp` to create a `REVD_MERGE_PASSTHRU` node
an invalid REV node is created as REV only takes one vector operand,
but both operands from the shuffle vector are copied to the new REV node.
This is not an issue in practice as the extra operand is ignored.
These issues were found by the verification added in #140472.
Part of #140472.
Note: Test changes only result in the vxi64 lowering matching the vxf64
lowering.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 29 +++++++++----------
.../AArch64/sve-fixed-length-permute-rev.ll | 3 +-
...streaming-mode-fixed-length-permute-rev.ll | 4 +--
3 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 293292d47dd48..99122e2897285 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -29773,11 +29773,18 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
return convertFromScalableVector(DAG, VT, Op);
}
+ auto lowerToRevMergePassthru = [&](unsigned Opcode, SDValue Vec, EVT NewVT) {
+ auto Pg = getPredicateForVector(DAG, DL, NewVT);
+ SDValue RevOp = DAG.getNode(ISD::BITCAST, DL, NewVT, Vec);
+ auto Rev =
+ DAG.getNode(Opcode, DL, NewVT, Pg, RevOp, DAG.getUNDEF(ContainerVT));
+ auto Cast = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Rev);
+ return convertFromScalableVector(DAG, VT, Cast);
+ };
+
unsigned EltSize = VT.getScalarSizeInBits();
for (unsigned LaneSize : {64U, 32U, 16U}) {
if (isREVMask(ShuffleMask, EltSize, VT.getVectorNumElements(), LaneSize)) {
- EVT NewVT =
- getPackedSVEVectorVT(EVT::getIntegerVT(*DAG.getContext(), LaneSize));
unsigned RevOp;
if (EltSize == 8)
RevOp = AArch64ISD::BSWAP_MERGE_PASSTHRU;
@@ -29785,24 +29792,16 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
RevOp = AArch64ISD::REVH_MERGE_PASSTHRU;
else
RevOp = AArch64ISD::REVW_MERGE_PASSTHRU;
-
- Op = DAG.getNode(ISD::BITCAST, DL, NewVT, Op1);
- Op = LowerToPredicatedOp(Op, DAG, RevOp);
- Op = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Op);
- return convertFromScalableVector(DAG, VT, Op);
+ EVT NewVT =
+ getPackedSVEVectorVT(EVT::getIntegerVT(*DAG.getContext(), LaneSize));
+ return lowerToRevMergePassthru(RevOp, Op1, NewVT);
}
}
if (Subtarget->hasSVE2p1() && EltSize == 64 &&
isREVMask(ShuffleMask, EltSize, VT.getVectorNumElements(), 128)) {
- if (!VT.isFloatingPoint())
- return LowerToPredicatedOp(Op, DAG, AArch64ISD::REVD_MERGE_PASSTHRU);
-
- EVT NewVT = getPackedSVEVectorVT(EVT::getIntegerVT(*DAG.getContext(), 64));
- Op = DAG.getNode(ISD::BITCAST, DL, NewVT, Op1);
- Op = LowerToPredicatedOp(Op, DAG, AArch64ISD::REVD_MERGE_PASSTHRU);
- Op = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Op);
- return convertFromScalableVector(DAG, VT, Op);
+ return lowerToRevMergePassthru(AArch64ISD::REVD_MERGE_PASSTHRU, Op1,
+ ContainerVT);
}
unsigned WhichResult;
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll
index 0cda4d94444e9..faf82d4945b3d 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll
@@ -213,8 +213,9 @@ define void @test_revdv4i64_sve2p1(ptr %a) #2 {
; CHECK-LABEL: test_revdv4i64_sve2p1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
+; CHECK-NEXT: ptrue p1.d
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: revd z0.q, p0/m, z0.q
+; CHECK-NEXT: revd z0.q, p1/m, z0.q
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%tmp1 = load <4 x i64>, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll
index c364abf2916e8..d8f83834a1bca 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll
@@ -677,7 +677,7 @@ define void @test_revdv4i64_sve2p1(ptr %a) #1 {
; CHECK-LABEL: test_revdv4i64_sve2p1:
; CHECK: // %bb.0:
; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: revd z0.q, p0/m, z0.q
; CHECK-NEXT: revd z1.q, p0/m, z1.q
; CHECK-NEXT: stp q0, q1, [x0]
@@ -686,7 +686,7 @@ define void @test_revdv4i64_sve2p1(ptr %a) #1 {
; NONEON-NOSVE-LABEL: test_revdv4i64_sve2p1:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT: ptrue p0.d, vl2
+; NONEON-NOSVE-NEXT: ptrue p0.d
; NONEON-NOSVE-NEXT: revd z0.q, p0/m, z0.q
; NONEON-NOSVE-NEXT: revd z1.q, p0/m, z1.q
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
>From 721759ea63154938decd0e6bf7a1a82910e2e95e Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell at arm.com>
Date: Tue, 20 May 2025 13:15:57 +0000
Subject: [PATCH 2/2] Tweak types
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 14 +++++++-------
.../AArch64/sve-fixed-length-permute-rev.ll | 6 ++----
.../sve-streaming-mode-fixed-length-permute-rev.ll | 8 ++++----
3 files changed, 13 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 99122e2897285..b0f24b4e28de0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -29773,11 +29773,11 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
return convertFromScalableVector(DAG, VT, Op);
}
- auto lowerToRevMergePassthru = [&](unsigned Opcode, SDValue Vec, EVT NewVT) {
- auto Pg = getPredicateForVector(DAG, DL, NewVT);
- SDValue RevOp = DAG.getNode(ISD::BITCAST, DL, NewVT, Vec);
- auto Rev =
- DAG.getNode(Opcode, DL, NewVT, Pg, RevOp, DAG.getUNDEF(ContainerVT));
+ auto lowerToRevMergePassthru = [&](unsigned Opcode, SDValue Vec,
+ EVT PredVecVT, EVT RevVT) {
+ auto Pg = getPredicateForVector(DAG, DL, PredVecVT);
+ SDValue RevOp = DAG.getNode(ISD::BITCAST, DL, RevVT, Vec);
+ auto Rev = DAG.getNode(Opcode, DL, RevVT, Pg, RevOp, DAG.getUNDEF(RevVT));
auto Cast = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Rev);
return convertFromScalableVector(DAG, VT, Cast);
};
@@ -29794,13 +29794,13 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
RevOp = AArch64ISD::REVW_MERGE_PASSTHRU;
EVT NewVT =
getPackedSVEVectorVT(EVT::getIntegerVT(*DAG.getContext(), LaneSize));
- return lowerToRevMergePassthru(RevOp, Op1, NewVT);
+ return lowerToRevMergePassthru(RevOp, Op1, NewVT, NewVT);
}
}
if (Subtarget->hasSVE2p1() && EltSize == 64 &&
isREVMask(ShuffleMask, EltSize, VT.getVectorNumElements(), 128)) {
- return lowerToRevMergePassthru(AArch64ISD::REVD_MERGE_PASSTHRU, Op1,
+ return lowerToRevMergePassthru(AArch64ISD::REVD_MERGE_PASSTHRU, Op1, VT,
ContainerVT);
}
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll
index faf82d4945b3d..42f9bec94721e 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll
@@ -213,9 +213,8 @@ define void @test_revdv4i64_sve2p1(ptr %a) #2 {
; CHECK-LABEL: test_revdv4i64_sve2p1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
-; CHECK-NEXT: ptrue p1.d
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: revd z0.q, p1/m, z0.q
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%tmp1 = load <4 x i64>, ptr %a
@@ -228,9 +227,8 @@ define void @test_revdv4f64_sve2p1(ptr %a) #2 {
; CHECK-LABEL: test_revdv4f64_sve2p1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
-; CHECK-NEXT: ptrue p1.d
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: revd z0.q, p1/m, z0.q
+; CHECK-NEXT: revd z0.q, p0/m, z0.q
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%tmp1 = load <4 x double>, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll
index d8f83834a1bca..890bc721128ff 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll
@@ -677,7 +677,7 @@ define void @test_revdv4i64_sve2p1(ptr %a) #1 {
; CHECK-LABEL: test_revdv4i64_sve2p1:
; CHECK: // %bb.0:
; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: revd z0.q, p0/m, z0.q
; CHECK-NEXT: revd z1.q, p0/m, z1.q
; CHECK-NEXT: stp q0, q1, [x0]
@@ -686,7 +686,7 @@ define void @test_revdv4i64_sve2p1(ptr %a) #1 {
; NONEON-NOSVE-LABEL: test_revdv4i64_sve2p1:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT: ptrue p0.d
+; NONEON-NOSVE-NEXT: ptrue p0.d, vl2
; NONEON-NOSVE-NEXT: revd z0.q, p0/m, z0.q
; NONEON-NOSVE-NEXT: revd z1.q, p0/m, z1.q
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
@@ -701,7 +701,7 @@ define void @test_revdv4f64_sve2p1(ptr %a) #1 {
; CHECK-LABEL: test_revdv4f64_sve2p1:
; CHECK: // %bb.0:
; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: revd z0.q, p0/m, z0.q
; CHECK-NEXT: revd z1.q, p0/m, z1.q
; CHECK-NEXT: stp q0, q1, [x0]
@@ -710,7 +710,7 @@ define void @test_revdv4f64_sve2p1(ptr %a) #1 {
; NONEON-NOSVE-LABEL: test_revdv4f64_sve2p1:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q0, q1, [x0]
-; NONEON-NOSVE-NEXT: ptrue p0.d
+; NONEON-NOSVE-NEXT: ptrue p0.d, vl2
; NONEON-NOSVE-NEXT: revd z0.q, p0/m, z0.q
; NONEON-NOSVE-NEXT: revd z1.q, p0/m, z1.q
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
More information about the llvm-commits
mailing list