[llvm] d15fff6 - Re-apply '[AArch64] Enable "sink-and-fold" in MachineSink by default (#67432)'
Momchil Velikov via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 19 05:19:56 PDT 2023
Author: Momchil Velikov
Date: 2023-10-19T13:18:25+01:00
New Revision: d15fff6c69c93670b6f63e0f0fd91fcdf69b6702
URL: https://github.com/llvm/llvm-project/commit/d15fff6c69c93670b6f63e0f0fd91fcdf69b6702
DIFF: https://github.com/llvm/llvm-project/commit/d15fff6c69c93670b6f63e0f0fd91fcdf69b6702.diff
LOG: Re-apply '[AArch64] Enable "sink-and-fold" in MachineSink by default (#67432)'
This reverts revert 19505072123e43eccf528b660973067b5c9b4a26.
An issue was fixed in bea3684944c0d7962cd53ab77aad756cfee76b7c
and some newly appeared tests updated.
Added:
Modified:
llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
llvm/test/CodeGen/AArch64/aarch64-mulv.ll
llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
llvm/test/CodeGen/AArch64/machine-sink-cache-invalidation.ll
llvm/test/CodeGen/AArch64/sink-and-fold.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 3d818c76bd4b7d7..fcc30a7cfceaf47 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -200,7 +200,7 @@ static cl::opt<bool> EnableGISelLoadStoreOptPostLegal(
static cl::opt<bool>
EnableSinkFold("aarch64-enable-sink-fold",
cl::desc("Enable sinking and folding of instruction copies"),
- cl::init(false), cl::Hidden);
+ cl::init(true), cl::Hidden);
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64Target() {
// Register the target.
diff --git a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
index 995023e80c44bef..819bd4f4c42a831 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-mulv.ll
@@ -584,10 +584,11 @@ define i128 @mulv_v2i128(<2 x i128> %a) {
; CHECK-GI-LABEL: mulv_v2i128:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: mul x9, x0, x3
-; CHECK-GI-NEXT: umulh x8, x0, x2
+; CHECK-GI-NEXT: mul x8, x0, x2
+; CHECK-GI-NEXT: umulh x10, x0, x2
; CHECK-GI-NEXT: madd x9, x1, x2, x9
-; CHECK-GI-NEXT: mul x0, x0, x2
-; CHECK-GI-NEXT: add x1, x9, x8
+; CHECK-GI-NEXT: mov x0, x8
+; CHECK-GI-NEXT: add x1, x9, x10
; CHECK-GI-NEXT: ret
entry:
%arg1 = call i128 @llvm.vector.reduce.mul.v2i128(<2 x i128> %a)
diff --git a/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll b/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
index e40063def477ccb..2765e22617f33c8 100644
--- a/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-indexed-memory.ll
@@ -727,25 +727,11 @@ define ptr @pretrunc64to8(ptr %ptr, i64 %spacing) {
; Pre-indexed loads
;-----
define ptr @preidxf64(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidxf64:
-; CHECK64: ; %bb.0:
-; CHECK64-NEXT: ldr d0, [x0, #8]!
-; CHECK64-NEXT: str d0, [x1]
-; CHECK64-NEXT: ret
-;
-; GISEL-LABEL: preidxf64:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #8
-; GISEL-NEXT: ldr d0, [x0, #8]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str d0, [x1]
-; GISEL-NEXT: ret
-;
-; CHECK32-LABEL: preidxf64:
-; CHECK32: ; %bb.0:
-; CHECK32-NEXT: ldr d0, [x0, #8]!
-; CHECK32-NEXT: str d0, [x1]
-; CHECK32-NEXT: ret
+; CHECK-LABEL: preidxf64:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: ldr d0, [x0, #8]!
+; CHECK-NEXT: str d0, [x1]
+; CHECK-NEXT: ret
%ptr = getelementptr inbounds double, ptr %src, i64 1
%tmp = load double, ptr %ptr, align 4
store double %tmp, ptr %out, align 4
@@ -753,25 +739,11 @@ define ptr @preidxf64(ptr %src, ptr %out) {
}
define ptr @preidxf32(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidxf32:
-; CHECK64: ; %bb.0:
-; CHECK64-NEXT: ldr s0, [x0, #4]!
-; CHECK64-NEXT: str s0, [x1]
-; CHECK64-NEXT: ret
-;
-; GISEL-LABEL: preidxf32:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #4
-; GISEL-NEXT: ldr s0, [x0, #4]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str s0, [x1]
-; GISEL-NEXT: ret
-;
-; CHECK32-LABEL: preidxf32:
-; CHECK32: ; %bb.0:
-; CHECK32-NEXT: ldr s0, [x0, #4]!
-; CHECK32-NEXT: str s0, [x1]
-; CHECK32-NEXT: ret
+; CHECK-LABEL: preidxf32:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: ldr s0, [x0, #4]!
+; CHECK-NEXT: str s0, [x1]
+; CHECK-NEXT: ret
%ptr = getelementptr inbounds float, ptr %src, i64 1
%tmp = load float, ptr %ptr, align 4
store float %tmp, ptr %out, align 4
@@ -787,9 +759,8 @@ define ptr @preidxf16(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidxf16:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #2
; GISEL-NEXT: ldr h0, [x0, #2]
-; GISEL-NEXT: mov x0, x8
+; GISEL-NEXT: add x0, x0, #2
; GISEL-NEXT: str h0, [x1]
; GISEL-NEXT: ret
;
@@ -805,25 +776,11 @@ define ptr @preidxf16(ptr %src, ptr %out) {
}
define ptr @preidx64(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx64:
-; CHECK64: ; %bb.0:
-; CHECK64-NEXT: ldr x8, [x0, #8]!
-; CHECK64-NEXT: str x8, [x1]
-; CHECK64-NEXT: ret
-;
-; GISEL-LABEL: preidx64:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #8
-; GISEL-NEXT: ldr x9, [x0, #8]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str x9, [x1]
-; GISEL-NEXT: ret
-;
-; CHECK32-LABEL: preidx64:
-; CHECK32: ; %bb.0:
-; CHECK32-NEXT: ldr x8, [x0, #8]!
-; CHECK32-NEXT: str x8, [x1]
-; CHECK32-NEXT: ret
+; CHECK-LABEL: preidx64:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: ldr x8, [x0, #8]!
+; CHECK-NEXT: str x8, [x1]
+; CHECK-NEXT: ret
%ptr = getelementptr inbounds i64, ptr %src, i64 1
%tmp = load i64, ptr %ptr, align 4
store i64 %tmp, ptr %out, align 4
@@ -831,25 +788,11 @@ define ptr @preidx64(ptr %src, ptr %out) {
}
define ptr @preidx32(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx32:
-; CHECK64: ; %bb.0:
-; CHECK64-NEXT: ldr w8, [x0, #4]!
-; CHECK64-NEXT: str w8, [x1]
-; CHECK64-NEXT: ret
-;
-; GISEL-LABEL: preidx32:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #4
-; GISEL-NEXT: ldr w9, [x0, #4]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str w9, [x1]
-; GISEL-NEXT: ret
-;
-; CHECK32-LABEL: preidx32:
-; CHECK32: ; %bb.0:
-; CHECK32-NEXT: ldr w8, [x0, #4]!
-; CHECK32-NEXT: str w8, [x1]
-; CHECK32-NEXT: ret
+; CHECK-LABEL: preidx32:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: ldr w8, [x0, #4]!
+; CHECK-NEXT: str w8, [x1]
+; CHECK-NEXT: ret
%ptr = getelementptr inbounds i32, ptr %src, i64 1
%tmp = load i32, ptr %ptr, align 4
store i32 %tmp, ptr %out, align 4
@@ -857,25 +800,11 @@ define ptr @preidx32(ptr %src, ptr %out) {
}
define ptr @preidx16zext32(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx16zext32:
-; CHECK64: ; %bb.0:
-; CHECK64-NEXT: ldrh w8, [x0, #2]!
-; CHECK64-NEXT: str w8, [x1]
-; CHECK64-NEXT: ret
-;
-; GISEL-LABEL: preidx16zext32:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #2
-; GISEL-NEXT: ldrh w9, [x0, #2]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str w9, [x1]
-; GISEL-NEXT: ret
-;
-; CHECK32-LABEL: preidx16zext32:
-; CHECK32: ; %bb.0:
-; CHECK32-NEXT: ldrh w8, [x0, #2]!
-; CHECK32-NEXT: str w8, [x1]
-; CHECK32-NEXT: ret
+; CHECK-LABEL: preidx16zext32:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: ldrh w8, [x0, #2]!
+; CHECK-NEXT: str w8, [x1]
+; CHECK-NEXT: ret
%ptr = getelementptr inbounds i16, ptr %src, i64 1
%tmp = load i16, ptr %ptr, align 4
%ext = zext i16 %tmp to i32
@@ -884,25 +813,11 @@ define ptr @preidx16zext32(ptr %src, ptr %out) {
}
define ptr @preidx16zext64(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx16zext64:
-; CHECK64: ; %bb.0:
-; CHECK64-NEXT: ldrh w8, [x0, #2]!
-; CHECK64-NEXT: str x8, [x1]
-; CHECK64-NEXT: ret
-;
-; GISEL-LABEL: preidx16zext64:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #2
-; GISEL-NEXT: ldrh w9, [x0, #2]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str x9, [x1]
-; GISEL-NEXT: ret
-;
-; CHECK32-LABEL: preidx16zext64:
-; CHECK32: ; %bb.0:
-; CHECK32-NEXT: ldrh w8, [x0, #2]!
-; CHECK32-NEXT: str x8, [x1]
-; CHECK32-NEXT: ret
+; CHECK-LABEL: preidx16zext64:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: ldrh w8, [x0, #2]!
+; CHECK-NEXT: str x8, [x1]
+; CHECK-NEXT: ret
%ptr = getelementptr inbounds i16, ptr %src, i64 1
%tmp = load i16, ptr %ptr, align 4
%ext = zext i16 %tmp to i64
@@ -911,25 +826,11 @@ define ptr @preidx16zext64(ptr %src, ptr %out) {
}
define ptr @preidx8zext32(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx8zext32:
-; CHECK64: ; %bb.0:
-; CHECK64-NEXT: ldrb w8, [x0, #1]!
-; CHECK64-NEXT: str w8, [x1]
-; CHECK64-NEXT: ret
-;
-; GISEL-LABEL: preidx8zext32:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #1
-; GISEL-NEXT: ldrb w9, [x0, #1]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str w9, [x1]
-; GISEL-NEXT: ret
-;
-; CHECK32-LABEL: preidx8zext32:
-; CHECK32: ; %bb.0:
-; CHECK32-NEXT: ldrb w8, [x0, #1]!
-; CHECK32-NEXT: str w8, [x1]
-; CHECK32-NEXT: ret
+; CHECK-LABEL: preidx8zext32:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: ldrb w8, [x0, #1]!
+; CHECK-NEXT: str w8, [x1]
+; CHECK-NEXT: ret
%ptr = getelementptr inbounds i8, ptr %src, i64 1
%tmp = load i8, ptr %ptr, align 4
%ext = zext i8 %tmp to i32
@@ -938,25 +839,11 @@ define ptr @preidx8zext32(ptr %src, ptr %out) {
}
define ptr @preidx8zext64(ptr %src, ptr %out) {
-; CHECK64-LABEL: preidx8zext64:
-; CHECK64: ; %bb.0:
-; CHECK64-NEXT: ldrb w8, [x0, #1]!
-; CHECK64-NEXT: str x8, [x1]
-; CHECK64-NEXT: ret
-;
-; GISEL-LABEL: preidx8zext64:
-; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #1
-; GISEL-NEXT: ldrb w9, [x0, #1]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str x9, [x1]
-; GISEL-NEXT: ret
-;
-; CHECK32-LABEL: preidx8zext64:
-; CHECK32: ; %bb.0:
-; CHECK32-NEXT: ldrb w8, [x0, #1]!
-; CHECK32-NEXT: str x8, [x1]
-; CHECK32-NEXT: ret
+; CHECK-LABEL: preidx8zext64:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: ldrb w8, [x0, #1]!
+; CHECK-NEXT: str x8, [x1]
+; CHECK-NEXT: ret
%ptr = getelementptr inbounds i8, ptr %src, i64 1
%tmp = load i8, ptr %ptr, align 4
%ext = zext i8 %tmp to i64
@@ -973,10 +860,9 @@ define ptr @preidx32sext64(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidx32sext64:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #4
-; GISEL-NEXT: ldrsw x9, [x0, #4]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str x9, [x1]
+; GISEL-NEXT: ldrsw x8, [x0, #4]
+; GISEL-NEXT: add x0, x0, #4
+; GISEL-NEXT: str x8, [x1]
; GISEL-NEXT: ret
;
; CHECK32-LABEL: preidx32sext64:
@@ -1000,10 +886,9 @@ define ptr @preidx16sext32(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidx16sext32:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #2
-; GISEL-NEXT: ldrsh w9, [x0, #2]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str w9, [x1]
+; GISEL-NEXT: ldrsh w8, [x0, #2]
+; GISEL-NEXT: add x0, x0, #2
+; GISEL-NEXT: str w8, [x1]
; GISEL-NEXT: ret
;
; CHECK32-LABEL: preidx16sext32:
@@ -1027,10 +912,9 @@ define ptr @preidx16sext64(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidx16sext64:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #2
-; GISEL-NEXT: ldrsh x9, [x0, #2]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str x9, [x1]
+; GISEL-NEXT: ldrsh x8, [x0, #2]
+; GISEL-NEXT: add x0, x0, #2
+; GISEL-NEXT: str x8, [x1]
; GISEL-NEXT: ret
;
; CHECK32-LABEL: preidx16sext64:
@@ -1054,10 +938,9 @@ define ptr @preidx8sext32(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidx8sext32:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #1
-; GISEL-NEXT: ldrsb w9, [x0, #1]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str w9, [x1]
+; GISEL-NEXT: ldrsb w8, [x0, #1]
+; GISEL-NEXT: add x0, x0, #1
+; GISEL-NEXT: str w8, [x1]
; GISEL-NEXT: ret
;
; CHECK32-LABEL: preidx8sext32:
@@ -1081,10 +964,9 @@ define ptr @preidx8sext64(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidx8sext64:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #1
-; GISEL-NEXT: ldrsb x9, [x0, #1]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str x9, [x1]
+; GISEL-NEXT: ldrsb x8, [x0, #1]
+; GISEL-NEXT: add x0, x0, #1
+; GISEL-NEXT: str x8, [x1]
; GISEL-NEXT: ret
;
; CHECK32-LABEL: preidx8sext64:
@@ -1137,10 +1019,9 @@ define ptr @preidx32_sb(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidx32_sb:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #1
-; GISEL-NEXT: ldrsb w9, [x0, #1]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str w9, [x1]
+; GISEL-NEXT: ldrsb w8, [x0, #1]
+; GISEL-NEXT: add x0, x0, #1
+; GISEL-NEXT: str w8, [x1]
; GISEL-NEXT: ret
;
; CHECK32-LABEL: preidx32_sb:
@@ -1164,10 +1045,9 @@ define ptr @preidx32_sh(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidx32_sh:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #2
-; GISEL-NEXT: ldrsh w9, [x0, #2]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str w9, [x1]
+; GISEL-NEXT: ldrsh w8, [x0, #2]
+; GISEL-NEXT: add x0, x0, #2
+; GISEL-NEXT: str w8, [x1]
; GISEL-NEXT: ret
;
; CHECK32-LABEL: preidx32_sh:
@@ -1191,10 +1071,9 @@ define ptr @preidx64_sb(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidx64_sb:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #1
-; GISEL-NEXT: ldrsb x9, [x0, #1]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str x9, [x1]
+; GISEL-NEXT: ldrsb x8, [x0, #1]
+; GISEL-NEXT: add x0, x0, #1
+; GISEL-NEXT: str x8, [x1]
; GISEL-NEXT: ret
;
; CHECK32-LABEL: preidx64_sb:
@@ -1218,10 +1097,9 @@ define ptr @preidx64_sh(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidx64_sh:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #2
-; GISEL-NEXT: ldrsh x9, [x0, #2]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str x9, [x1]
+; GISEL-NEXT: ldrsh x8, [x0, #2]
+; GISEL-NEXT: add x0, x0, #2
+; GISEL-NEXT: str x8, [x1]
; GISEL-NEXT: ret
;
; CHECK32-LABEL: preidx64_sh:
@@ -1245,10 +1123,9 @@ define ptr @preidx64_sw(ptr %src, ptr %out) {
;
; GISEL-LABEL: preidx64_sw:
; GISEL: ; %bb.0:
-; GISEL-NEXT: add x8, x0, #4
-; GISEL-NEXT: ldrsw x9, [x0, #4]
-; GISEL-NEXT: mov x0, x8
-; GISEL-NEXT: str x9, [x1]
+; GISEL-NEXT: ldrsw x8, [x0, #4]
+; GISEL-NEXT: add x0, x0, #4
+; GISEL-NEXT: str x8, [x1]
; GISEL-NEXT: ret
;
; CHECK32-LABEL: preidx64_sw:
diff --git a/llvm/test/CodeGen/AArch64/machine-sink-cache-invalidation.ll b/llvm/test/CodeGen/AArch64/machine-sink-cache-invalidation.ll
index ad6fdb6f1f9b944..ce000021fb29bff 100644
--- a/llvm/test/CodeGen/AArch64/machine-sink-cache-invalidation.ll
+++ b/llvm/test/CodeGen/AArch64/machine-sink-cache-invalidation.ll
@@ -22,8 +22,7 @@ define i32 @nsis_BZ2_bzDecompress(ptr %pos.i, i1 %cmp661.not3117.i, i1 %exitcond
; CHECK-NEXT: // in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: mov x9, xzr
; CHECK-NEXT: ldrb w9, [x9]
-; CHECK-NEXT: add x9, x0, x9
-; CHECK-NEXT: strb wzr, [x9]
+; CHECK-NEXT: strb wzr, [x0, x9]
; CHECK-NEXT: b .LBB0_1
; CHECK-NEXT: .LBB0_4: // %for.end677.i
; CHECK-NEXT: mov w0, wzr
diff --git a/llvm/test/CodeGen/AArch64/sink-and-fold.ll b/llvm/test/CodeGen/AArch64/sink-and-fold.ll
index 632fdb391053121..52007221e12a7b5 100644
--- a/llvm/test/CodeGen/AArch64/sink-and-fold.ll
+++ b/llvm/test/CodeGen/AArch64/sink-and-fold.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -aarch64-enable-sink-fold=true < %s | FileCheck %s
+; RUN: llc < %s | FileCheck %s
target triple = "aarch64-linux"
declare i32 @use(...)
More information about the llvm-commits
mailing list