r309488 - [X86][AVX] Ensure vector non-temporal load/store intrinsics force pointer alignment (PR33830)
Hans Wennborg via cfe-commits
cfe-commits at lists.llvm.org
Mon Jul 31 10:31:01 PDT 2017
Thanks! r309588.
On Mon, Jul 31, 2017 at 10:25 AM, Craig Topper <craig.topper at gmail.com> wrote:
> We can merge all three patches to 5.0.
>
> ~Craig
>
> On Mon, Jul 31, 2017 at 9:59 AM, Hans Wennborg <hans at chromium.org> wrote:
>>
>> Craig, Simon asked for this to be merged to 5.0 together with r309382
>> and r309383 for PR33830. What do you think?
>>
>> On Sat, Jul 29, 2017 at 8:33 AM, Simon Pilgrim via cfe-commits
>> <cfe-commits at lists.llvm.org> wrote:
>> > Author: rksimon
>> > Date: Sat Jul 29 08:33:34 2017
>> > New Revision: 309488
>> >
>> > URL: http://llvm.org/viewvc/llvm-project?rev=309488&view=rev
>> > Log:
>> > [X86][AVX] Ensure vector non-temporal load/store intrinsics force
>> > pointer alignment (PR33830)
>> >
>> > Clang specifies a max type alignment of 16 bytes on darwin targets
>> > (annoyingly in the driver not via cc1), meaning that the builtin nontemporal
>> > stores don't correctly align the loads/stores to 32 or 64 bytes when
>> > required, resulting in lowering to temporal unaligned loads/stores.
>> >
>> > This patch casts the vectors to explicitly aligned types prior to the
>> > load/store to ensure that the require alignment is respected.
>> >
>> > Differential Revision: https://reviews.llvm.org/D35996
>> >
>> > Modified:
>> > cfe/trunk/lib/Headers/avx2intrin.h
>> > cfe/trunk/lib/Headers/avx512fintrin.h
>> > cfe/trunk/lib/Headers/avxintrin.h
>> > cfe/trunk/test/CodeGen/x86-nontemporal.c
>> >
>> > Modified: cfe/trunk/lib/Headers/avx2intrin.h
>> > URL:
>> > http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Headers/avx2intrin.h?rev=309488&r1=309487&r2=309488&view=diff
>> >
>> > ==============================================================================
>> > --- cfe/trunk/lib/Headers/avx2intrin.h (original)
>> > +++ cfe/trunk/lib/Headers/avx2intrin.h Sat Jul 29 08:33:34 2017
>> > @@ -832,7 +832,8 @@ _mm256_xor_si256(__m256i __a, __m256i __
>> > static __inline__ __m256i __DEFAULT_FN_ATTRS
>> > _mm256_stream_load_si256(__m256i const *__V)
>> > {
>> > - return (__m256i)__builtin_nontemporal_load((const __v4di *)__V);
>> > + typedef __v4di __v4di_aligned __attribute__((aligned(32)));
>> > + return (__m256i)__builtin_nontemporal_load((const __v4di_aligned
>> > *)__V);
>> > }
>> >
>> > static __inline__ __m128 __DEFAULT_FN_ATTRS
>> >
>> > Modified: cfe/trunk/lib/Headers/avx512fintrin.h
>> > URL:
>> > http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Headers/avx512fintrin.h?rev=309488&r1=309487&r2=309488&view=diff
>> >
>> > ==============================================================================
>> > --- cfe/trunk/lib/Headers/avx512fintrin.h (original)
>> > +++ cfe/trunk/lib/Headers/avx512fintrin.h Sat Jul 29 08:33:34 2017
>> > @@ -9035,25 +9035,29 @@ _mm512_kxor (__mmask16 __A, __mmask16 __
>> > static __inline__ void __DEFAULT_FN_ATTRS
>> > _mm512_stream_si512 (__m512i * __P, __m512i __A)
>> > {
>> > - __builtin_nontemporal_store((__v8di)__A, (__v8di*)__P);
>> > + typedef __v8di __v8di_aligned __attribute__((aligned(64)));
>> > + __builtin_nontemporal_store((__v8di_aligned)__A,
>> > (__v8di_aligned*)__P);
>> > }
>> >
>> > static __inline__ __m512i __DEFAULT_FN_ATTRS
>> > _mm512_stream_load_si512 (void *__P)
>> > {
>> > - return (__m512i) __builtin_nontemporal_load((const __v8di *)__P);
>> > + typedef __v8di __v8di_aligned __attribute__((aligned(64)));
>> > + return (__m512i) __builtin_nontemporal_load((const __v8di_aligned
>> > *)__P);
>> > }
>> >
>> > static __inline__ void __DEFAULT_FN_ATTRS
>> > _mm512_stream_pd (double *__P, __m512d __A)
>> > {
>> > - __builtin_nontemporal_store((__v8df)__A, (__v8df*)__P);
>> > + typedef __v8df __v8df_aligned __attribute__((aligned(64)));
>> > + __builtin_nontemporal_store((__v8df_aligned)__A,
>> > (__v8df_aligned*)__P);
>> > }
>> >
>> > static __inline__ void __DEFAULT_FN_ATTRS
>> > _mm512_stream_ps (float *__P, __m512 __A)
>> > {
>> > - __builtin_nontemporal_store((__v16sf)__A, (__v16sf*)__P);
>> > + typedef __v16sf __v16sf_aligned __attribute__((aligned(64)));
>> > + __builtin_nontemporal_store((__v16sf_aligned)__A,
>> > (__v16sf_aligned*)__P);
>> > }
>> >
>> > static __inline__ __m512d __DEFAULT_FN_ATTRS
>> >
>> > Modified: cfe/trunk/lib/Headers/avxintrin.h
>> > URL:
>> > http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Headers/avxintrin.h?rev=309488&r1=309487&r2=309488&view=diff
>> >
>> > ==============================================================================
>> > --- cfe/trunk/lib/Headers/avxintrin.h (original)
>> > +++ cfe/trunk/lib/Headers/avxintrin.h Sat Jul 29 08:33:34 2017
>> > @@ -3590,7 +3590,8 @@ _mm_maskstore_ps(float *__p, __m128i __m
>> > static __inline void __DEFAULT_FN_ATTRS
>> > _mm256_stream_si256(__m256i *__a, __m256i __b)
>> > {
>> > - __builtin_nontemporal_store((__v4di)__b, (__v4di*)__a);
>> > + typedef __v4di __v4di_aligned __attribute__((aligned(32)));
>> > + __builtin_nontemporal_store((__v4di_aligned)__b,
>> > (__v4di_aligned*)__a);
>> > }
>> >
>> > /// \brief Moves double-precision values from a 256-bit vector of [4 x
>> > double]
>> > @@ -3609,7 +3610,8 @@ _mm256_stream_si256(__m256i *__a, __m256
>> > static __inline void __DEFAULT_FN_ATTRS
>> > _mm256_stream_pd(double *__a, __m256d __b)
>> > {
>> > - __builtin_nontemporal_store((__v4df)__b, (__v4df*)__a);
>> > + typedef __v4df __v4df_aligned __attribute__((aligned(32)));
>> > + __builtin_nontemporal_store((__v4df_aligned)__b,
>> > (__v4df_aligned*)__a);
>> > }
>> >
>> > /// \brief Moves single-precision floating point values from a 256-bit
>> > vector
>> > @@ -3629,7 +3631,8 @@ _mm256_stream_pd(double *__a, __m256d __
>> > static __inline void __DEFAULT_FN_ATTRS
>> > _mm256_stream_ps(float *__p, __m256 __a)
>> > {
>> > - __builtin_nontemporal_store((__v8sf)__a, (__v8sf*)__p);
>> > + typedef __v8sf __v8sf_aligned __attribute__((aligned(32)));
>> > + __builtin_nontemporal_store((__v8sf_aligned)__a,
>> > (__v8sf_aligned*)__p);
>> > }
>> >
>> > /* Create vectors */
>> >
>> > Modified: cfe/trunk/test/CodeGen/x86-nontemporal.c
>> > URL:
>> > http://llvm.org/viewvc/llvm-project/cfe/trunk/test/CodeGen/x86-nontemporal.c?rev=309488&r1=309487&r2=309488&view=diff
>> >
>> > ==============================================================================
>> > --- cfe/trunk/test/CodeGen/x86-nontemporal.c (original)
>> > +++ cfe/trunk/test/CodeGen/x86-nontemporal.c Sat Jul 29 08:33:34 2017
>> > @@ -1,12 +1,12 @@
>> > // RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin
>> > -target-feature +sse4.1 -target-feature +avx -target-feature +avx2
>> > -target-feature +avx512f -emit-llvm -o - -Wall -Werror | FileCheck %s
>> > --check-prefix=CHECK
>> > // RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin
>> > -target-feature +sse4.1 -target-feature +avx -target-feature +avx2
>> > -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror |
>> > FileCheck %s --check-prefix=CHECK
>> >
>> > -// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin
>> > -target-feature +sse4.1 -target-feature +avx -target-feature +avx2
>> > -target-feature +avx512f -emit-llvm -o - -Wall -Werror -fmax-type-align=16 |
>> > FileCheck %s --check-prefix=CHECK16
>> > -// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin
>> > -target-feature +sse4.1 -target-feature +avx -target-feature +avx2
>> > -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror
>> > -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK16
>> > +// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin
>> > -target-feature +sse4.1 -target-feature +avx -target-feature +avx2
>> > -target-feature +avx512f -emit-llvm -o - -Wall -Werror -fmax-type-align=16 |
>> > FileCheck %s --check-prefix=CHECK
>> > +// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin
>> > -target-feature +sse4.1 -target-feature +avx -target-feature +avx2
>> > -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror
>> > -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK
>> >
>> > #include <x86intrin.h>
>> >
>> > -// FIXME: (PR33830) Tests showing failure to correctly align
>> > non-temporal load/stores on darwin targets where fmax-type-align is set to
>> > 16.
>> > +// (PR33830) Tests ensure the correct alignment of non-temporal
>> > load/stores on darwin targets where fmax-type-align is set to 16.
>> >
>> > //
>> > // 128-bit vectors
>> > @@ -43,36 +43,24 @@ __m128i test_mm_stream_load_si128(__m128
>> > void test_mm256_stream_pd(double* A, __m256d B) {
>> > // CHECK-LABEL: test_mm256_stream_pd
>> > // CHECK: store <4 x double> %{{.*}}, <4 x double>* %{{.*}}, align
>> > 32, !nontemporal
>> > -
>> > - // CHECK16-LABEL: test_mm256_stream_pd
>> > - // CHECK16: store <4 x double> %{{.*}}, <4 x double>* %{{.*}}, align
>> > 16, !nontemporal
>> > _mm256_stream_pd(A, B);
>> > }
>> >
>> > void test_mm256_stream_ps(float* A, __m256 B) {
>> > // CHECK-LABEL: test_mm256_stream_ps
>> > // CHECK: store <8 x float> %{{.*}}, <8 x float>* %{{.*}}, align 32,
>> > !nontemporal
>> > -
>> > - // CHECK16-LABEL: test_mm256_stream_ps
>> > - // CHECK16: store <8 x float> %{{.*}}, <8 x float>* %{{.*}}, align
>> > 16, !nontemporal
>> > _mm256_stream_ps(A, B);
>> > }
>> >
>> > void test_mm256_stream_si256(__m256i* A, __m256i B) {
>> > // CHECK-LABEL: test_mm256_stream_si256
>> > // CHECK: store <4 x i64> %{{.*}}, <4 x i64>* %{{.*}}, align 32,
>> > !nontemporal
>> > -
>> > - // CHECK16-LABEL: test_mm256_stream_si256
>> > - // CHECK16: store <4 x i64> %{{.*}}, <4 x i64>* %{{.*}}, align 16,
>> > !nontemporal
>> > _mm256_stream_si256(A, B);
>> > }
>> >
>> > __m256i test_mm256_stream_load_si256(__m256i const *A) {
>> > // CHECK-LABEL: test_mm256_stream_load_si256
>> > // CHECK: load <4 x i64>, <4 x i64>* %{{.*}}, align 32, !nontemporal
>> > -
>> > - // CHECK16-LABEL: test_mm256_stream_load_si256
>> > - // CHECK16: load <4 x i64>, <4 x i64>* %{{.*}}, align 16,
>> > !nontemporal
>> > return _mm256_stream_load_si256(A);
>> > }
>> >
>> > @@ -83,35 +71,23 @@ __m256i test_mm256_stream_load_si256(__m
>> > void test_mm512_stream_pd(double* A, __m512d B) {
>> > // CHECK-LABEL: test_mm512_stream_pd
>> > // CHECK: store <8 x double> %{{.*}}, <8 x double>* %{{.*}}, align
>> > 64, !nontemporal
>> > -
>> > - // CHECK16-LABEL: test_mm512_stream_pd
>> > - // CHECK16: store <8 x double> %{{.*}}, <8 x double>* %{{.*}}, align
>> > 16, !nontemporal
>> > _mm512_stream_pd(A, B);
>> > }
>> >
>> > void test_mm512_stream_ps(float* A, __m512 B) {
>> > // CHECK-LABEL: test_mm512_stream_ps
>> > // CHECK: store <16 x float> %{{.*}}, <16 x float>* %{{.*}}, align
>> > 64, !nontemporal
>> > -
>> > - // CHECK16-LABEL: test_mm512_stream_ps
>> > - // CHECK16: store <16 x float> %{{.*}}, <16 x float>* %{{.*}}, align
>> > 16, !nontemporal
>> > _mm512_stream_ps(A, B);
>> > }
>> >
>> > void test_mm512_stream_si512(__m512i* A, __m512i B) {
>> > // CHECK-LABEL: test_mm512_stream_si512
>> > // CHECK: store <8 x i64> %{{.*}}, <8 x i64>* %{{.*}}, align 64,
>> > !nontemporal
>> > -
>> > - // CHECK16-LABEL: test_mm512_stream_si512
>> > - // CHECK16: store <8 x i64> %{{.*}}, <8 x i64>* %{{.*}}, align 16,
>> > !nontemporal
>> > _mm512_stream_si512(A, B);
>> > }
>> >
>> > __m512i test_mm512_stream_load_si512(void *A) {
>> > // CHECK-LABEL: test_mm512_stream_load_si512
>> > // CHECK: load <8 x i64>, <8 x i64>* %{{.*}}, align 64, !nontemporal
>> > -
>> > - // CHECK16-LABEL: test_mm512_stream_load_si512
>> > - // CHECK16: load <8 x i64>, <8 x i64>* %{{.*}}, align 16,
>> > !nontemporal
>> > return _mm512_stream_load_si512(A);
>> > }
>> >
>> >
>> > _______________________________________________
>> > cfe-commits mailing list
>> > cfe-commits at lists.llvm.org
>> > http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
>
>
More information about the cfe-commits
mailing list