[llvm-branch-commits] [llvm] ca4bf58 - [AMDGPU] Support unaligned flat scratch in TLI
Stanislav Mekhanoshin via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Dec 22 16:36:07 PST 2020
Author: Stanislav Mekhanoshin
Date: 2020-12-22T16:12:31-08:00
New Revision: ca4bf58e4ee5951473a861716193063c5ef83e9a
URL: https://github.com/llvm/llvm-project/commit/ca4bf58e4ee5951473a861716193063c5ef83e9a
DIFF: https://github.com/llvm/llvm-project/commit/ca4bf58e4ee5951473a861716193063c5ef83e9a.diff
LOG: [AMDGPU] Support unaligned flat scratch in TLI
Adjust SITargetLowering::allowsMisalignedMemoryAccessesImpl for
unaligned flat scratch support. Mostly needed for global isel.
Differential Revision: https://reviews.llvm.org/D93669
Added:
Modified:
llvm/lib/Target/AMDGPU/SIISelLowering.cpp
llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 5fb1924bdd9f..81fdfa0343b3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1470,12 +1470,21 @@ bool SITargetLowering::allowsMisalignedMemoryAccessesImpl(
}
}
+ if (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS) {
+ bool AlignedBy4 = Alignment >= Align(4);
+ if (IsFast)
+ *IsFast = AlignedBy4;
+
+ return AlignedBy4 ||
+ Subtarget->enableFlatScratch() ||
+ Subtarget->hasUnalignedScratchAccess();
+ }
+
// FIXME: We have to be conservative here and assume that flat operations
// will access scratch. If we had access to the IR function, then we
// could determine if any private memory was used in the function.
- if (!Subtarget->hasUnalignedScratchAccess() &&
- (AddrSpace == AMDGPUAS::PRIVATE_ADDRESS ||
- AddrSpace == AMDGPUAS::FLAT_ADDRESS)) {
+ if (AddrSpace == AMDGPUAS::FLAT_ADDRESS &&
+ !Subtarget->hasUnalignedScratchAccess()) {
bool AlignedBy4 = Alignment >= Align(4);
if (IsFast)
*IsFast = AlignedBy4;
diff --git a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
index 271f6c703980..8e37b413ddf5 100644
--- a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
@@ -271,16 +271,9 @@ define amdgpu_kernel void @vload2_private(i16 addrspace(1)* nocapture readonly %
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: scratch_store_short off, v0, vcc_hi offset:8
; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0
-; FLATSCR-NEXT: scratch_load_ushort v0, off, vcc_hi offset:4
+; FLATSCR-NEXT: scratch_load_dword v0, off, vcc_hi offset:4
; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0
-; FLATSCR-NEXT: scratch_load_ushort v3, off, vcc_hi offset:6
-; FLATSCR-NEXT: s_mov_b32 vcc_hi, 0
-; FLATSCR-NEXT: s_waitcnt vmcnt(1)
-; FLATSCR-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: v_mov_b32_e32 v1, v3
-; FLATSCR-NEXT: scratch_load_short_d16_hi v1, off, vcc_hi offset:8
-; FLATSCR-NEXT: v_lshl_or_b32 v0, v3, 16, v0
+; FLATSCR-NEXT: scratch_load_dword v1, off, vcc_hi offset:6
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
; FLATSCR-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll b/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
index 5d5cfd318edf..645eead8c297 100644
--- a/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/unaligned-load-store.ll
@@ -1,6 +1,7 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=ALIGNED %s
-; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=+unaligned-access-mode -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=UNALIGNED %s
-; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -check-prefix=SI -check-prefix=ALIGNED %s
+; RUN: llc -march=amdgcn -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,ALIGNED %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -mattr=+unaligned-access-mode -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,UNALIGNED %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs< %s | FileCheck -check-prefixes=SI,MUBUF,ALIGNED %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-enable-flat-scratch -verify-machineinstrs < %s | FileCheck -check-prefixes=SI,FLATSCR,ALIGNED %s
; SI-LABEL: {{^}}local_unaligned_load_store_i16:
; SI: ds_read_u8
@@ -602,64 +603,70 @@ define amdgpu_kernel void @local_store_align1_v16i8(<16 x i8> addrspace(3)* %out
}
; SI-LABEL: {{^}}private_load_align1_f64:
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
-; SI: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; MUBUF: buffer_load_ubyte
+; FLATSCR: scratch_load_dwordx2
define double @private_load_align1_f64(double addrspace(5)* %in) {
%x = load double, double addrspace(5)* %in, align 1
ret double %x
}
; SI-LABEL: {{^}}private_store_align1_f64:
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
-; SI: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; MUBUF: buffer_store_byte
+; FLATSCR: scratch_store_dwordx2
define void @private_store_align1_f64(double addrspace(5)* %out, double %x) #0 {
store double %x, double addrspace(5)* %out, align 1
ret void
}
; SI-LABEL: {{^}}private_load_align4_f64:
-; SI: buffer_load_dword
-; SI: buffer_load_dword
+; MUBUF: buffer_load_dword
+; MUBUF: buffer_load_dword
+; FLATSCR: scratch_load_dwordx2
define double @private_load_align4_f64(double addrspace(5)* %in) {
%x = load double, double addrspace(5)* %in, align 4
ret double %x
}
; SI-LABEL: {{^}}private_store_align4_f64:
-; SI: buffer_store_dword
-; SI: buffer_store_dword
+; MUBUF: buffer_store_dword
+; MUBUF: buffer_store_dword
+; FLATSCR: scratch_store_dwordx2
define void @private_store_align4_f64(double addrspace(5)* %out, double %x) #0 {
store double %x, double addrspace(5)* %out, align 4
ret void
}
; SI-LABEL: {{^}}private_load_align2_f64:
-; SI: buffer_load_ushort
-; SI: buffer_load_ushort
-; SI: buffer_load_ushort
-; SI: buffer_load_ushort
+; MUBUF: buffer_load_ushort
+; MUBUF: buffer_load_ushort
+; MUBUF: buffer_load_ushort
+; MUBUF: buffer_load_ushort
+; FLATSCR: scratch_load_dwordx2
define double @private_load_align2_f64(double addrspace(5)* %in) {
%x = load double, double addrspace(5)* %in, align 2
ret double %x
}
; SI-LABEL: {{^}}private_store_align2_f64:
-; SI: buffer_store_short
-; SI: buffer_store_short
-; SI: buffer_store_short
-; SI: buffer_store_short
+; MUBUF: buffer_store_short
+; MUBUF: buffer_store_short
+; MUBUF: buffer_store_short
+; MUBUF: buffer_store_short
+; FLATSCR: scratch_store_dwordx2
define void @private_store_align2_f64(double addrspace(5)* %out, double %x) #0 {
store double %x, double addrspace(5)* %out, align 2
ret void
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
index a46f4d4175b7..5bbacbf6a034 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
@@ -9,9 +9,6 @@ target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:3
; ALL-LABEL: @load_unknown_offset_align1_i8(
; ALL: alloca [128 x i8], align 1
; UNALIGNED: load <2 x i8>, <2 x i8> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
-
-; ALIGNED: load i8, i8 addrspace(5)* %ptr0, align 1{{$}}
-; ALIGNED: load i8, i8 addrspace(5)* %ptr1, align 1{{$}}
define amdgpu_kernel void @load_unknown_offset_align1_i8(i8 addrspace(1)* noalias %out, i32 %offset) #0 {
%alloca = alloca [128 x i8], align 1, addrspace(5)
%ptr0 = getelementptr inbounds [128 x i8], [128 x i8] addrspace(5)* %alloca, i32 0, i32 %offset
@@ -60,13 +57,11 @@ define amdgpu_kernel void @load_unknown_offset_align1_i32(i32 addrspace(1)* noal
ret void
}
-; FIXME: Should always increase alignment of the load
; Make sure alloca alignment isn't decreased
; ALL-LABEL: @load_alloca16_unknown_offset_align1_i32(
; ALL: alloca [128 x i32], align 16
-; UNALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 1{{$}}
-; ALIGNED: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}}
+; ALL: load <2 x i32>, <2 x i32> addrspace(5)* %{{[0-9]+}}, align 4{{$}}
define amdgpu_kernel void @load_alloca16_unknown_offset_align1_i32(i32 addrspace(1)* noalias %out, i32 %offset) #0 {
%alloca = alloca [128 x i32], align 16, addrspace(5)
%ptr0 = getelementptr inbounds [128 x i32], [128 x i32] addrspace(5)* %alloca, i32 0, i32 %offset
@@ -128,11 +123,8 @@ define amdgpu_kernel void @store_unknown_offset_align1_i32(i32 addrspace(1)* noa
}
; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32(
-; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5)
-; ALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 4
-
-; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5)
-; UNALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 1
+; ALL: %alloca = alloca [8 x i32], align 4, addrspace(5)
+; ALL: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, <4 x i32> addrspace(5)* %1, align 4
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() {
%alloca = alloca [8 x i32], align 1, addrspace(5)
%out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)*
@@ -148,11 +140,8 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() {
}
; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8(
-; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5)
-; ALIGNED: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 4
-
-; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5)
-; UNALIGNED: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 1
+; ALL: %alloca = alloca [8 x i8], align 4, addrspace(5)
+; ALL: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, <4 x i8> addrspace(5)* %1, align 4
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() {
%alloca = alloca [8 x i8], align 1, addrspace(5)
%out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)*
@@ -168,11 +157,8 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() {
}
; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i32(
-; ALIGNED: %alloca = alloca [8 x i32], align 4, addrspace(5)
-; ALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4
-
-; UNALIGNED: %alloca = alloca [8 x i32], align 1, addrspace(5)
-; UNALIGNED: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 1
+; ALL: %alloca = alloca [8 x i32], align 4, addrspace(5)
+; ALL: load <4 x i32>, <4 x i32> addrspace(5)* %1, align 4
define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() {
%alloca = alloca [8 x i32], align 1, addrspace(5)
%out = bitcast [8 x i32] addrspace(5)* %alloca to i32 addrspace(5)*
@@ -188,11 +174,8 @@ define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() {
}
; ALL-LABEL: @merge_private_load_4_vector_elts_loads_v4i8(
-; ALIGNED: %alloca = alloca [8 x i8], align 4, addrspace(5)
-; ALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4
-
-; UNALIGNED: %alloca = alloca [8 x i8], align 1, addrspace(5)
-; UNALIGNED: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 1
+; ALL: %alloca = alloca [8 x i8], align 4, addrspace(5)
+; ALL: load <4 x i8>, <4 x i8> addrspace(5)* %1, align 4
define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i8() {
%alloca = alloca [8 x i8], align 1, addrspace(5)
%out = bitcast [8 x i8] addrspace(5)* %alloca to i8 addrspace(5)*
More information about the llvm-branch-commits
mailing list