[llvm] af81d8e - [AArch64][CodeGen] Add patterns for small negative VScale const (#89607)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 24 04:44:19 PDT 2024
Author: Allen
Date: 2024-04-24T19:44:15+08:00
New Revision: af81d8ec224900de7a4d2c96a675269499b55a0c
URL: https://github.com/llvm/llvm-project/commit/af81d8ec224900de7a4d2c96a675269499b55a0c
DIFF: https://github.com/llvm/llvm-project/commit/af81d8ec224900de7a4d2c96a675269499b55a0c.diff
LOG: [AArch64][CodeGen] Add patterns for small negative VScale const (#89607)
On AArch64, rdvl can accept a nagative value, while cntd/cntw/cnth can't.
As we do support VScale with a negative multiply value, so we did not limit
the negative value and instead took the hit of having the extra patterns according PR88108.
Also add NoUseScalarIncVL to avoid affecting patterns works for -mattr=+use-scalar-inc-vl
Fix https://github.com/llvm/llvm-project/issues/84620
Added:
Modified:
llvm/lib/Target/AArch64/AArch64InstrInfo.td
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
llvm/test/CodeGen/AArch64/sve-vl-arith.ll
llvm/test/CodeGen/AArch64/vscale-and-sve-cnt-demandedbits.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index fb18d1c63ae80d..a7abb58064a535 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -315,6 +315,8 @@ def UseNegativeImmediates
def UseScalarIncVL : Predicate<"Subtarget->useScalarIncVL()">;
+def NoUseScalarIncVL : Predicate<"!Subtarget->useScalarIncVL()">;
+
def UseSVEFPLD1R : Predicate<"!Subtarget->noSVEFPLD1R()">;
def IsNeonAvailable : Predicate<"Subtarget->isNeonAvailable()">;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 705fa523a8b8f2..525ae79da99624 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -2517,6 +2517,23 @@ let Predicates = [HasSVEorSME] in {
def : Pat<(vscale (sve_cntd_imm_neg i32:$imm)), (SUBXrs XZR, (CNTD_XPiI 31, $imm), 0)>;
}
+ // Add NoUseScalarIncVL to avoid affecting for patterns with UseScalarIncVL
+ let Predicates = [HasSVEorSME, NoUseScalarIncVL] in {
+ def : Pat<(add GPR64:$op, (vscale (sve_cnth_imm_neg i32:$imm))),
+ (SUBXrs GPR64:$op, (CNTH_XPiI 31, $imm), 0)>;
+ def : Pat<(add GPR64:$op, (vscale (sve_cntw_imm_neg i32:$imm))),
+ (SUBXrs GPR64:$op, (CNTW_XPiI 31, $imm), 0)>;
+ def : Pat<(add GPR64:$op, (vscale (sve_cntd_imm_neg i32:$imm))),
+ (SUBXrs GPR64:$op, (CNTD_XPiI 31, $imm), 0)>;
+
+ def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_cnth_imm_neg i32:$imm))))),
+ (SUBSWrr GPR32:$op, (EXTRACT_SUBREG (CNTH_XPiI 31, $imm), sub_32))>;
+ def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_cntw_imm_neg i32:$imm))))),
+ (SUBSWrr GPR32:$op, (EXTRACT_SUBREG (CNTW_XPiI 31, $imm), sub_32))>;
+ def : Pat<(add GPR32:$op, (i32 (trunc (vscale (sve_cntd_imm_neg i32:$imm))))),
+ (SUBSWrr GPR32:$op, (EXTRACT_SUBREG (CNTD_XPiI 31, $imm), sub_32))>;
+ }
+
let AddedComplexity = 5 in {
def : Pat<(nxv8i16 (add ZPR:$op, (nxv8i16 (splat_vector (i32 (trunc (vscale (sve_cnth_imm i32:$imm)))))))),
(INCH_ZPiI ZPR:$op, 31, $imm)>;
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
index 664d99a3627b58..5bef95910d9060 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions-scalable.ll
@@ -17,11 +17,11 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
; CHECK-NEXT: mov z1.d, #0 // =0x0
; CHECK-NEXT: cntd x9
; CHECK-NEXT: ptrue p1.b
-; CHECK-NEXT: neg x9, x9
-; CHECK-NEXT: mov w10, #100 // =0x64
+; CHECK-NEXT: neg x10, x9
+; CHECK-NEXT: mov w11, #100 // =0x64
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: and x10, x9, x10
+; CHECK-NEXT: and x10, x10, x11
; CHECK-NEXT: rdvl x11, #2
; CHECK-NEXT: zip2 z0.d, z1.d, z1.d
; CHECK-NEXT: zip1 z1.d, z1.d, z1.d
@@ -33,7 +33,7 @@ define %"class.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) {
; CHECK-NEXT: ld1d { z3.d }, p0/z, [x12, #1, mul vl]
; CHECK-NEXT: ld1b { z4.b }, p1/z, [x1, x8]
; CHECK-NEXT: ld1d { z5.d }, p0/z, [x13, #1, mul vl]
-; CHECK-NEXT: adds x10, x10, x9
+; CHECK-NEXT: subs x10, x10, x9
; CHECK-NEXT: add x8, x8, x11
; CHECK-NEXT: fcmla z1.d, p0/m, z4.d, z2.d, #0
; CHECK-NEXT: fcmla z0.d, p0/m, z5.d, z3.d, #0
@@ -106,12 +106,12 @@ define %"class.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
; CHECK-NEXT: cntd x9
; CHECK-NEXT: fmov d2, #2.00000000
; CHECK-NEXT: ptrue p0.d, vl1
-; CHECK-NEXT: neg x9, x9
+; CHECK-NEXT: neg x10, x9
; CHECK-NEXT: ptrue p1.b
-; CHECK-NEXT: mov w10, #100 // =0x64
+; CHECK-NEXT: mov w11, #100 // =0x64
; CHECK-NEXT: mov x8, xzr
; CHECK-NEXT: sel z3.d, p0, z0.d, z1.d
-; CHECK-NEXT: and x10, x9, x10
+; CHECK-NEXT: and x10, x10, x11
; CHECK-NEXT: rdvl x11, #2
; CHECK-NEXT: mov z1.d, p0/m, z2.d
; CHECK-NEXT: ptrue p0.d
@@ -125,7 +125,7 @@ define %"class.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) {
; CHECK-NEXT: ld1d { z3.d }, p0/z, [x12, #1, mul vl]
; CHECK-NEXT: ld1b { z4.b }, p1/z, [x1, x8]
; CHECK-NEXT: ld1d { z5.d }, p0/z, [x13, #1, mul vl]
-; CHECK-NEXT: adds x10, x10, x9
+; CHECK-NEXT: subs x10, x10, x9
; CHECK-NEXT: add x8, x8, x11
; CHECK-NEXT: fcmla z1.d, p0/m, z4.d, z2.d, #0
; CHECK-NEXT: fcmla z0.d, p0/m, z5.d, z3.d, #0
@@ -191,13 +191,13 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov z1.d, #0 // =0x0
; CHECK-NEXT: cntw x9
-; CHECK-NEXT: mov w10, #1000 // =0x3e8
-; CHECK-NEXT: neg x9, x9
+; CHECK-NEXT: mov w11, #1000 // =0x3e8
+; CHECK-NEXT: neg x10, x9
; CHECK-NEXT: rdvl x12, #2
; CHECK-NEXT: ptrue p1.b
; CHECK-NEXT: ptrue p0.d
; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: and x10, x9, x10
+; CHECK-NEXT: and x10, x10, x11
; CHECK-NEXT: zip2 z0.d, z1.d, z1.d
; CHECK-NEXT: zip1 z1.d, z1.d, z1.d
; CHECK-NEXT: add x11, x1, x12
@@ -219,7 +219,7 @@ define %"class.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) {
; CHECK-NEXT: ld1d { z17.d }, p0/z, [x15, #1, mul vl]
; CHECK-NEXT: ld1b { z18.b }, p1/z, [x11, x8]
; CHECK-NEXT: ld1d { z19.d }, p0/z, [x17, #1, mul vl]
-; CHECK-NEXT: adds x10, x10, x9
+; CHECK-NEXT: subs x10, x10, x9
; CHECK-NEXT: add x8, x8, x13
; CHECK-NEXT: fcmla z1.d, p0/m, z7.d, z4.d, #0
; CHECK-NEXT: fcmla z0.d, p0/m, z16.d, z5.d, #0
diff --git a/llvm/test/CodeGen/AArch64/sve-vl-arith.ll b/llvm/test/CodeGen/AArch64/sve-vl-arith.ll
index dd4294c8d3bdcc..de2af590acd1e2 100644
--- a/llvm/test/CodeGen/AArch64/sve-vl-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vl-arith.ll
@@ -186,8 +186,8 @@ define i64 @incd_scalar_i64(i64 %a) {
define i64 @decb_scalar_i64(i64 %a) {
; NO_SCALAR_INC-LABEL: decb_scalar_i64:
; NO_SCALAR_INC: // %bb.0:
-; NO_SCALAR_INC-NEXT: rdvl x8, #-2
-; NO_SCALAR_INC-NEXT: add x0, x0, x8
+; NO_SCALAR_INC-NEXT: cnth x8, all, mul #4
+; NO_SCALAR_INC-NEXT: sub x0, x0, x8
; NO_SCALAR_INC-NEXT: ret
;
; CHECK-LABEL: decb_scalar_i64:
@@ -204,8 +204,7 @@ define i64 @dech_scalar_i64(i64 %a) {
; NO_SCALAR_INC-LABEL: dech_scalar_i64:
; NO_SCALAR_INC: // %bb.0:
; NO_SCALAR_INC-NEXT: cnth x8, all, mul #3
-; NO_SCALAR_INC-NEXT: neg x8, x8
-; NO_SCALAR_INC-NEXT: add x0, x0, x8
+; NO_SCALAR_INC-NEXT: sub x0, x0, x8
; NO_SCALAR_INC-NEXT: ret
;
; CHECK-LABEL: dech_scalar_i64:
@@ -222,8 +221,7 @@ define i64 @decw_scalar_i64(i64 %a) {
; NO_SCALAR_INC-LABEL: decw_scalar_i64:
; NO_SCALAR_INC: // %bb.0:
; NO_SCALAR_INC-NEXT: cntw x8, all, mul #3
-; NO_SCALAR_INC-NEXT: neg x8, x8
-; NO_SCALAR_INC-NEXT: add x0, x0, x8
+; NO_SCALAR_INC-NEXT: sub x0, x0, x8
; NO_SCALAR_INC-NEXT: ret
;
; CHECK-LABEL: decw_scalar_i64:
@@ -240,8 +238,7 @@ define i64 @decd_scalar_i64(i64 %a) {
; NO_SCALAR_INC-LABEL: decd_scalar_i64:
; NO_SCALAR_INC: // %bb.0:
; NO_SCALAR_INC-NEXT: cntd x8, all, mul #3
-; NO_SCALAR_INC-NEXT: neg x8, x8
-; NO_SCALAR_INC-NEXT: add x0, x0, x8
+; NO_SCALAR_INC-NEXT: sub x0, x0, x8
; NO_SCALAR_INC-NEXT: ret
;
; CHECK-LABEL: decd_scalar_i64:
@@ -345,8 +342,8 @@ define i32 @incd_scalar_i32(i32 %a) {
define i32 @decb_scalar_i32(i32 %a) {
; NO_SCALAR_INC-LABEL: decb_scalar_i32:
; NO_SCALAR_INC: // %bb.0:
-; NO_SCALAR_INC-NEXT: rdvl x8, #-4
-; NO_SCALAR_INC-NEXT: add w0, w0, w8
+; NO_SCALAR_INC-NEXT: cnth x8, all, mul #8
+; NO_SCALAR_INC-NEXT: sub w0, w0, w8
; NO_SCALAR_INC-NEXT: ret
;
; CHECK-LABEL: decb_scalar_i32:
@@ -367,8 +364,7 @@ define i32 @dech_scalar_i32(i32 %a) {
; NO_SCALAR_INC-LABEL: dech_scalar_i32:
; NO_SCALAR_INC: // %bb.0:
; NO_SCALAR_INC-NEXT: cnth x8
-; NO_SCALAR_INC-NEXT: neg x8, x8
-; NO_SCALAR_INC-NEXT: add w0, w0, w8
+; NO_SCALAR_INC-NEXT: sub w0, w0, w8
; NO_SCALAR_INC-NEXT: ret
;
; CHECK-LABEL: dech_scalar_i32:
@@ -389,8 +385,7 @@ define i32 @decw_scalar_i32(i32 %a) {
; NO_SCALAR_INC-LABEL: decw_scalar_i32:
; NO_SCALAR_INC: // %bb.0:
; NO_SCALAR_INC-NEXT: cntw x8
-; NO_SCALAR_INC-NEXT: neg x8, x8
-; NO_SCALAR_INC-NEXT: add w0, w0, w8
+; NO_SCALAR_INC-NEXT: sub w0, w0, w8
; NO_SCALAR_INC-NEXT: ret
;
; CHECK-LABEL: decw_scalar_i32:
@@ -411,8 +406,7 @@ define i32 @decd_scalar_i32(i32 %a) {
; NO_SCALAR_INC-LABEL: decd_scalar_i32:
; NO_SCALAR_INC: // %bb.0:
; NO_SCALAR_INC-NEXT: cntd x8
-; NO_SCALAR_INC-NEXT: neg x8, x8
-; NO_SCALAR_INC-NEXT: add w0, w0, w8
+; NO_SCALAR_INC-NEXT: sub w0, w0, w8
; NO_SCALAR_INC-NEXT: ret
;
; CHECK-LABEL: decd_scalar_i32:
diff --git a/llvm/test/CodeGen/AArch64/vscale-and-sve-cnt-demandedbits.ll b/llvm/test/CodeGen/AArch64/vscale-and-sve-cnt-demandedbits.ll
index dbdab799c83522..9572778484f8d3 100644
--- a/llvm/test/CodeGen/AArch64/vscale-and-sve-cnt-demandedbits.ll
+++ b/llvm/test/CodeGen/AArch64/vscale-and-sve-cnt-demandedbits.ll
@@ -194,7 +194,7 @@ define i32 @vscale_with_multiplier() vscale_range(1,16) {
; CHECK-LABEL: vscale_with_multiplier:
; CHECK: // %bb.0:
; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: mov w9, #5
+; CHECK-NEXT: mov w9, #5 // =0x5
; CHECK-NEXT: lsr x8, x8, #4
; CHECK-NEXT: mul x8, x8, x9
; CHECK-NEXT: and w9, w8, #0x3f
@@ -212,7 +212,7 @@ define i32 @vscale_with_negative_multiplier() vscale_range(1,16) {
; CHECK-LABEL: vscale_with_negative_multiplier:
; CHECK: // %bb.0:
; CHECK-NEXT: rdvl x8, #1
-; CHECK-NEXT: mov x9, #-5
+; CHECK-NEXT: mov x9, #-5 // =0xfffffffffffffffb
; CHECK-NEXT: lsr x8, x8, #4
; CHECK-NEXT: mul x8, x8, x9
; CHECK-NEXT: and w9, w8, #0xffffffc0
@@ -230,9 +230,9 @@ define i32 @pow2_vscale_with_negative_multiplier() vscale_range(1,16) {
; CHECK-LABEL: pow2_vscale_with_negative_multiplier:
; CHECK: // %bb.0:
; CHECK-NEXT: cntd x8
-; CHECK-NEXT: neg x8, x8
-; CHECK-NEXT: orr w9, w8, #0xfffffff0
-; CHECK-NEXT: add w0, w8, w9
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: orr w9, w9, #0xfffffff0
+; CHECK-NEXT: sub w0, w9, w8
; CHECK-NEXT: ret
%vscale = call i32 @llvm.vscale.i32()
%mul = mul i32 %vscale, -2
More information about the llvm-commits
mailing list