[llvm] f42e321 - [AArch64] Use FMOVDr for clearing upper bits (#83107)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Feb 27 11:45:47 PST 2024
Author: David Green
Date: 2024-02-27T19:45:43Z
New Revision: f42e321b9fb54300c4450e699cd3cc453e994b58
URL: https://github.com/llvm/llvm-project/commit/f42e321b9fb54300c4450e699cd3cc453e994b58
DIFF: https://github.com/llvm/llvm-project/commit/f42e321b9fb54300c4450e699cd3cc453e994b58.diff
LOG: [AArch64] Use FMOVDr for clearing upper bits (#83107)
This adds some tablegen patterns for generating FMOVDr from concat(X,
zeroes), as the FMOV will implicitly zero the upper bits of the
register. An extra AArch64MIPeepholeOpt is needed to make sure we can
remove the FMOV in the same way we would remove the insert code.
Added:
Modified:
llvm/lib/Target/AArch64/AArch64InstrInfo.td
llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
llvm/test/CodeGen/AArch64/implicitly-set-zero-high-64-bits.ll
llvm/test/CodeGen/AArch64/vecreduce-add.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index e73bc0d89e4c9d..b01a8cd00025f8 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -6667,31 +6667,29 @@ def : Pat<(vector_extract (v8bf16 V128:$Rn), VectorIndexH:$idx),
// All concat_vectors operations are canonicalised to act on i64 vectors for
// AArch64. In the general case we need an instruction, which had just as well be
// INS.
-class ConcatPat<ValueType DstTy, ValueType SrcTy>
- : Pat<(DstTy (concat_vectors (SrcTy V64:$Rd), V64:$Rn)),
- (INSvi64lane (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), 1,
- (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rn, dsub), 0)>;
-
-def : ConcatPat<v2i64, v1i64>;
-def : ConcatPat<v2f64, v1f64>;
-def : ConcatPat<v4i32, v2i32>;
-def : ConcatPat<v4f32, v2f32>;
-def : ConcatPat<v8i16, v4i16>;
-def : ConcatPat<v8f16, v4f16>;
-def : ConcatPat<v8bf16, v4bf16>;
-def : ConcatPat<v16i8, v8i8>;
-
-// If the high lanes are undef, though, we can just ignore them:
-class ConcatUndefPat<ValueType DstTy, ValueType SrcTy>
- : Pat<(DstTy (concat_vectors (SrcTy V64:$Rn), undef)),
- (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rn, dsub)>;
-
-def : ConcatUndefPat<v2i64, v1i64>;
-def : ConcatUndefPat<v2f64, v1f64>;
-def : ConcatUndefPat<v4i32, v2i32>;
-def : ConcatUndefPat<v4f32, v2f32>;
-def : ConcatUndefPat<v8i16, v4i16>;
-def : ConcatUndefPat<v16i8, v8i8>;
+multiclass ConcatPat<ValueType DstTy, ValueType SrcTy> {
+ def : Pat<(DstTy (concat_vectors (SrcTy V64:$Rd), V64:$Rn)),
+ (INSvi64lane (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), 1,
+ (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rn, dsub), 0)>;
+
+ // If the high lanes are zero we can instead emit a d->d register mov, which
+ // will implicitly clear the upper bits.
+ def : Pat<(DstTy (concat_vectors (SrcTy V64:$Rn), immAllZerosV)),
+ (SUBREG_TO_REG (i64 0), (FMOVDr V64:$Rn), dsub)>;
+
+ // If the high lanes are undef we can just ignore them:
+ def : Pat<(DstTy (concat_vectors (SrcTy V64:$Rn), undef)),
+ (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rn, dsub)>;
+}
+
+defm : ConcatPat<v2i64, v1i64>;
+defm : ConcatPat<v2f64, v1f64>;
+defm : ConcatPat<v4i32, v2i32>;
+defm : ConcatPat<v4f32, v2f32>;
+defm : ConcatPat<v8i16, v4i16>;
+defm : ConcatPat<v8f16, v4f16>;
+defm : ConcatPat<v8bf16, v4bf16>;
+defm : ConcatPat<v16i8, v8i8>;
//----------------------------------------------------------------------------
// AdvSIMD across lanes instructions
diff --git a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
index 87aa3b98d93826..6865850cf04feb 100644
--- a/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MIPeepholeOpt.cpp
@@ -127,6 +127,7 @@ struct AArch64MIPeepholeOpt : public MachineFunctionPass {
bool visitINSERT(MachineInstr &MI);
bool visitINSviGPR(MachineInstr &MI, unsigned Opc);
bool visitINSvi64lane(MachineInstr &MI);
+ bool visitFMOVDr(MachineInstr &MI);
bool runOnMachineFunction(MachineFunction &MF) override;
StringRef getPassName() const override {
@@ -670,6 +671,23 @@ bool AArch64MIPeepholeOpt::visitINSvi64lane(MachineInstr &MI) {
return true;
}
+bool AArch64MIPeepholeOpt::visitFMOVDr(MachineInstr &MI) {
+ // An FMOVDr sets the high 64-bits to zero implicitly, similar to ORR for GPR.
+ MachineInstr *Low64MI = MRI->getUniqueVRegDef(MI.getOperand(1).getReg());
+ if (!Low64MI || !is64bitDefwithZeroHigh64bit(Low64MI, MRI))
+ return false;
+
+ // Let's remove MIs for high 64-bits.
+ Register OldDef = MI.getOperand(0).getReg();
+ Register NewDef = MI.getOperand(1).getReg();
+ MRI->constrainRegClass(NewDef, MRI->getRegClass(OldDef));
+ MRI->replaceRegWith(OldDef, NewDef);
+ LLVM_DEBUG(dbgs() << "Removed: " << MI << "\n");
+ MI.eraseFromParent();
+
+ return true;
+}
+
bool AArch64MIPeepholeOpt::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
@@ -748,6 +766,9 @@ bool AArch64MIPeepholeOpt::runOnMachineFunction(MachineFunction &MF) {
case AArch64::INSvi64lane:
Changed |= visitINSvi64lane(MI);
break;
+ case AArch64::FMOVDr:
+ Changed |= visitFMOVDr(MI);
+ break;
}
}
}
diff --git a/llvm/test/CodeGen/AArch64/implicitly-set-zero-high-64-bits.ll b/llvm/test/CodeGen/AArch64/implicitly-set-zero-high-64-bits.ll
index 1eb9eab1c21e63..a949eaac5cfa29 100644
--- a/llvm/test/CodeGen/AArch64/implicitly-set-zero-high-64-bits.ll
+++ b/llvm/test/CodeGen/AArch64/implicitly-set-zero-high-64-bits.ll
@@ -137,9 +137,7 @@ entry:
define <16 x i8> @insertzero_v8i8(<8 x i8> %a) {
; CHECK-LABEL: insertzero_v8i8:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: fmov d0, d0
; CHECK-NEXT: ret
entry:
%shuffle.i = shufflevector <8 x i8> %a, <8 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -149,9 +147,7 @@ entry:
define <8 x i16> @insertzero_v4i16(<4 x i16> %a) {
; CHECK-LABEL: insertzero_v4i16:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: fmov d0, d0
; CHECK-NEXT: ret
entry:
%shuffle.i = shufflevector <4 x i16> %a, <4 x i16> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -161,9 +157,7 @@ entry:
define <4 x i32> @insertzero_v2i32(<2 x i32> %a) {
; CHECK-LABEL: insertzero_v2i32:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: fmov d0, d0
; CHECK-NEXT: ret
entry:
%shuffle.i = shufflevector <2 x i32> %a, <2 x i32> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -173,9 +167,7 @@ entry:
define <2 x i64> @insertzero_v1i64(<1 x i64> %a) {
; CHECK-LABEL: insertzero_v1i64:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: fmov d0, d0
; CHECK-NEXT: ret
entry:
%shuffle.i = shufflevector <1 x i64> %a, <1 x i64> zeroinitializer, <2 x i32> <i32 0, i32 1>
@@ -185,9 +177,7 @@ entry:
define <8 x half> @insertzero_v4f16(<4 x half> %a) {
; CHECK-LABEL: insertzero_v4f16:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi d1, #0000000000000000
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: fmov d0, d0
; CHECK-NEXT: ret
entry:
%shuffle.i = shufflevector <4 x half> %a, <4 x half> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -210,9 +200,7 @@ entry:
define <4 x float> @insertzero_v2f32(<2 x float> %a) {
; CHECK-LABEL: insertzero_v2f32:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi d1, #0000000000000000
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: fmov d0, d0
; CHECK-NEXT: ret
entry:
%shuffle.i = shufflevector <2 x float> %a, <2 x float> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -222,9 +210,7 @@ entry:
define <2 x double> @insertzero_v1f64(<1 x double> %a) {
; CHECK-LABEL: insertzero_v1f64:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi d1, #0000000000000000
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: fmov d0, d0
; CHECK-NEXT: ret
entry:
%shuffle.i = shufflevector <1 x double> %a, <1 x double> zeroinitializer, <2 x i32> <i32 0, i32 1>
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
index 86dd1bdd511eb3..66b49466cc7361 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-add.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -2182,8 +2182,8 @@ define i32 @test_udot_v24i8(ptr %p1, ptr %p2) {
; CHECK-GI-DOT-NEXT: ldr b5, [x0, #15]
; CHECK-GI-DOT-NEXT: mov v2.b[14], v6.b[0]
; CHECK-GI-DOT-NEXT: ldr b6, [x1, #15]
-; CHECK-GI-DOT-NEXT: mov v3.d[1], v0.d[0]
-; CHECK-GI-DOT-NEXT: mov v4.d[1], v0.d[0]
+; CHECK-GI-DOT-NEXT: fmov d3, d3
+; CHECK-GI-DOT-NEXT: fmov d4, d4
; CHECK-GI-DOT-NEXT: mov v1.b[15], v5.b[0]
; CHECK-GI-DOT-NEXT: movi v5.2d, #0000000000000000
; CHECK-GI-DOT-NEXT: mov v2.b[15], v6.b[0]
@@ -2760,8 +2760,8 @@ define i32 @test_sdot_v24i8(ptr %p1, ptr %p2) {
; CHECK-GI-DOT-NEXT: ldr b5, [x0, #15]
; CHECK-GI-DOT-NEXT: mov v2.b[14], v6.b[0]
; CHECK-GI-DOT-NEXT: ldr b6, [x1, #15]
-; CHECK-GI-DOT-NEXT: mov v3.d[1], v0.d[0]
-; CHECK-GI-DOT-NEXT: mov v4.d[1], v0.d[0]
+; CHECK-GI-DOT-NEXT: fmov d3, d3
+; CHECK-GI-DOT-NEXT: fmov d4, d4
; CHECK-GI-DOT-NEXT: mov v1.b[15], v5.b[0]
; CHECK-GI-DOT-NEXT: movi v5.2d, #0000000000000000
; CHECK-GI-DOT-NEXT: mov v2.b[15], v6.b[0]
More information about the llvm-commits
mailing list