[llvm-branch-commits] [cfe-branch] r309588 - Merging r309382:
Hans Wennborg via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jul 31 10:29:29 PDT 2017
Author: hans
Date: Mon Jul 31 10:29:29 2017
New Revision: 309588
URL: http://llvm.org/viewvc/llvm-project?rev=309588&view=rev
Log:
Merging r309382:
------------------------------------------------------------------------
r309382 | rksimon | 2017-07-28 06:47:02 -0700 (Fri, 28 Jul 2017) | 3 lines
[X86] Add tests showing inability of vector non-temporal load/store intrinsic to force pointer alignment (PR33830)
Clang specifies a max type alignment of 16 bytes on darwin targets, meaning that the builtin nontemporal stores don't correctly align the loads/stores to 32 or 64 bytes when required, resulting in lowering to temporal unaligned loads/stores.
------------------------------------------------------------------------
Merging r309383:
------------------------------------------------------------------------
r309383 | rksimon | 2017-07-28 07:01:51 -0700 (Fri, 28 Jul 2017) | 1 line
Strip trailing whitespace. NFCI.
------------------------------------------------------------------------
Merging r309488:
------------------------------------------------------------------------
r309488 | rksimon | 2017-07-29 08:33:34 -0700 (Sat, 29 Jul 2017) | 7 lines
[X86][AVX] Ensure vector non-temporal load/store intrinsics force pointer alignment (PR33830)
Clang specifies a max type alignment of 16 bytes on darwin targets (annoyingly in the driver not via cc1), meaning that the builtin nontemporal stores don't correctly align the loads/stores to 32 or 64 bytes when required, resulting in lowering to temporal unaligned loads/stores.
This patch casts the vectors to explicitly aligned types prior to the load/store to ensure that the require alignment is respected.
Differential Revision: https://reviews.llvm.org/D35996
------------------------------------------------------------------------
Added:
cfe/branches/release_50/test/CodeGen/x86-nontemporal.c
- copied, changed from r309382, cfe/trunk/test/CodeGen/x86-nontemporal.c
Modified:
cfe/branches/release_50/ (props changed)
cfe/branches/release_50/lib/Headers/avx2intrin.h
cfe/branches/release_50/lib/Headers/avx512fintrin.h
cfe/branches/release_50/lib/Headers/avxintrin.h
Propchange: cfe/branches/release_50/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jul 31 10:29:29 2017
@@ -1,4 +1,4 @@
/cfe/branches/type-system-rewrite:134693-134817
-/cfe/trunk:308455,308824,308897,309058,309112-309113,309226,309327,309503
+/cfe/trunk:308455,308824,308897,309058,309112-309113,309226,309327,309382-309383,309488,309503
/cfe/trunk/test:170344
/cfe/trunk/test/SemaTemplate:126920
Modified: cfe/branches/release_50/lib/Headers/avx2intrin.h
URL: http://llvm.org/viewvc/llvm-project/cfe/branches/release_50/lib/Headers/avx2intrin.h?rev=309588&r1=309587&r2=309588&view=diff
==============================================================================
--- cfe/branches/release_50/lib/Headers/avx2intrin.h (original)
+++ cfe/branches/release_50/lib/Headers/avx2intrin.h Mon Jul 31 10:29:29 2017
@@ -832,7 +832,8 @@ _mm256_xor_si256(__m256i __a, __m256i __
static __inline__ __m256i __DEFAULT_FN_ATTRS
_mm256_stream_load_si256(__m256i const *__V)
{
- return (__m256i)__builtin_nontemporal_load((const __v4di *)__V);
+ typedef __v4di __v4di_aligned __attribute__((aligned(32)));
+ return (__m256i)__builtin_nontemporal_load((const __v4di_aligned *)__V);
}
static __inline__ __m128 __DEFAULT_FN_ATTRS
Modified: cfe/branches/release_50/lib/Headers/avx512fintrin.h
URL: http://llvm.org/viewvc/llvm-project/cfe/branches/release_50/lib/Headers/avx512fintrin.h?rev=309588&r1=309587&r2=309588&view=diff
==============================================================================
--- cfe/branches/release_50/lib/Headers/avx512fintrin.h (original)
+++ cfe/branches/release_50/lib/Headers/avx512fintrin.h Mon Jul 31 10:29:29 2017
@@ -4289,7 +4289,7 @@ static __inline__ __m512i __DEFAULT_FN_A
_mm512_maskz_cvtps_epu32 ( __mmask16 __U, __m512 __A)
{
return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A,
- (__v16si)
+ (__v16si)
_mm512_setzero_si512 (),
(__mmask16) __U ,
_MM_FROUND_CUR_DIRECTION);
@@ -9035,25 +9035,29 @@ _mm512_kxor (__mmask16 __A, __mmask16 __
static __inline__ void __DEFAULT_FN_ATTRS
_mm512_stream_si512 (__m512i * __P, __m512i __A)
{
- __builtin_nontemporal_store((__v8di)__A, (__v8di*)__P);
+ typedef __v8di __v8di_aligned __attribute__((aligned(64)));
+ __builtin_nontemporal_store((__v8di_aligned)__A, (__v8di_aligned*)__P);
}
static __inline__ __m512i __DEFAULT_FN_ATTRS
_mm512_stream_load_si512 (void *__P)
{
- return (__m512i) __builtin_nontemporal_load((const __v8di *)__P);
+ typedef __v8di __v8di_aligned __attribute__((aligned(64)));
+ return (__m512i) __builtin_nontemporal_load((const __v8di_aligned *)__P);
}
static __inline__ void __DEFAULT_FN_ATTRS
_mm512_stream_pd (double *__P, __m512d __A)
{
- __builtin_nontemporal_store((__v8df)__A, (__v8df*)__P);
+ typedef __v8df __v8df_aligned __attribute__((aligned(64)));
+ __builtin_nontemporal_store((__v8df_aligned)__A, (__v8df_aligned*)__P);
}
static __inline__ void __DEFAULT_FN_ATTRS
_mm512_stream_ps (float *__P, __m512 __A)
{
- __builtin_nontemporal_store((__v16sf)__A, (__v16sf*)__P);
+ typedef __v16sf __v16sf_aligned __attribute__((aligned(64)));
+ __builtin_nontemporal_store((__v16sf_aligned)__A, (__v16sf_aligned*)__P);
}
static __inline__ __m512d __DEFAULT_FN_ATTRS
@@ -9217,39 +9221,39 @@ _mm512_maskz_moveldup_ps (__mmask16 __U,
static __inline__ __m128 __DEFAULT_FN_ATTRS
_mm_mask_move_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B)
{
- __m128 res = __A;
+ __m128 res = __A;
res[0] = (__U & 1) ? __B[0] : __W[0];
- return res;
+ return res;
}
static __inline__ __m128 __DEFAULT_FN_ATTRS
_mm_maskz_move_ss (__mmask8 __U, __m128 __A, __m128 __B)
{
- __m128 res = __A;
- res[0] = (__U & 1) ? __B[0] : 0;
- return res;
+ __m128 res = __A;
+ res[0] = (__U & 1) ? __B[0] : 0;
+ return res;
}
static __inline__ __m128d __DEFAULT_FN_ATTRS
_mm_mask_move_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B)
{
- __m128d res = __A;
+ __m128d res = __A;
res[0] = (__U & 1) ? __B[0] : __W[0];
- return res;
+ return res;
}
static __inline__ __m128d __DEFAULT_FN_ATTRS
_mm_maskz_move_sd (__mmask8 __U, __m128d __A, __m128d __B)
{
- __m128d res = __A;
- res[0] = (__U & 1) ? __B[0] : 0;
- return res;
+ __m128d res = __A;
+ res[0] = (__U & 1) ? __B[0] : 0;
+ return res;
}
static __inline__ void __DEFAULT_FN_ATTRS
_mm_mask_store_ss (float * __W, __mmask8 __U, __m128 __A)
{
- __builtin_ia32_storess128_mask ((__v16sf *)__W,
+ __builtin_ia32_storess128_mask ((__v16sf *)__W,
(__v16sf) _mm512_castps128_ps512(__A),
(__mmask16) __U & (__mmask16)1);
}
@@ -9257,7 +9261,7 @@ _mm_mask_store_ss (float * __W, __mmask8
static __inline__ void __DEFAULT_FN_ATTRS
_mm_mask_store_sd (double * __W, __mmask8 __U, __m128d __A)
{
- __builtin_ia32_storesd128_mask ((__v8df *)__W,
+ __builtin_ia32_storesd128_mask ((__v8df *)__W,
(__v8df) _mm512_castpd128_pd512(__A),
(__mmask8) __U & 1);
}
@@ -9606,7 +9610,7 @@ _mm_mask_cvtsd_ss (__m128 __W, __mmask8
{
return __builtin_ia32_cvtsd2ss_round_mask ((__v4sf)(__A),
(__v2df)(__B),
- (__v4sf)(__W),
+ (__v4sf)(__W),
(__mmask8)(__U), _MM_FROUND_CUR_DIRECTION);
}
@@ -9615,7 +9619,7 @@ _mm_maskz_cvtsd_ss (__mmask8 __U, __m128
{
return __builtin_ia32_cvtsd2ss_round_mask ((__v4sf)(__A),
(__v2df)(__B),
- (__v4sf)_mm_setzero_ps(),
+ (__v4sf)_mm_setzero_ps(),
(__mmask8)(__U), _MM_FROUND_CUR_DIRECTION);
}
@@ -9680,7 +9684,7 @@ _mm_mask_cvtss_sd (__m128d __W, __mmask8
return __builtin_ia32_cvtss2sd_round_mask((__v2df)(__A),
(__v4sf)(__B),
(__v2df)(__W),
- (__mmask8)(__U), _MM_FROUND_CUR_DIRECTION);
+ (__mmask8)(__U), _MM_FROUND_CUR_DIRECTION);
}
static __inline__ __m128d __DEFAULT_FN_ATTRS
@@ -9688,8 +9692,8 @@ _mm_maskz_cvtss_sd (__mmask8 __U, __m128
{
return __builtin_ia32_cvtss2sd_round_mask((__v2df)(__A),
(__v4sf)(__B),
- (__v2df)_mm_setzero_pd(),
- (__mmask8)(__U), _MM_FROUND_CUR_DIRECTION);
+ (__v2df)_mm_setzero_pd(),
+ (__mmask8)(__U), _MM_FROUND_CUR_DIRECTION);
}
static __inline__ __m128d __DEFAULT_FN_ATTRS
@@ -9935,7 +9939,7 @@ static __inline__ double __DEFAULT_FN_AT
}
// Vec512 - Vector with size 512.
-// Vec512Neutral - All vector elements set to the identity element.
+// Vec512Neutral - All vector elements set to the identity element.
// Identity element: {+,0},{*,1},{&,0xFFFFFFFFFFFFFFFF},{|,0}
// Operator - Can be one of following: +,*,&,|
// Mask - Intrinsic Mask
@@ -9965,19 +9969,19 @@ _mm512_mask_reduce_mul_epi64(__mmask8 __
static __inline__ long long __DEFAULT_FN_ATTRS
_mm512_mask_reduce_and_epi64(__mmask8 __M, __m512i __W) {
- _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_epi64(0xFFFFFFFFFFFFFFFF),
+ _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_epi64(0xFFFFFFFFFFFFFFFF),
&, __M, i, i, q);
}
static __inline__ long long __DEFAULT_FN_ATTRS
_mm512_mask_reduce_or_epi64(__mmask8 __M, __m512i __W) {
- _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_epi64(0), |, __M,
+ _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_epi64(0), |, __M,
i, i, q);
}
static __inline__ double __DEFAULT_FN_ATTRS
_mm512_mask_reduce_add_pd(__mmask8 __M, __m512d __W) {
- _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_pd(0), +, __M,
+ _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_pd(0), +, __M,
f, d, pd);
}
@@ -10039,17 +10043,17 @@ _mm512_reduce_add_epi32(__m512i __W) {
_mm512_reduce_operator_32bit(__W, +, i, i);
}
-static __inline__ int __DEFAULT_FN_ATTRS
+static __inline__ int __DEFAULT_FN_ATTRS
_mm512_reduce_mul_epi32(__m512i __W) {
_mm512_reduce_operator_32bit(__W, *, i, i);
}
-static __inline__ int __DEFAULT_FN_ATTRS
+static __inline__ int __DEFAULT_FN_ATTRS
_mm512_reduce_and_epi32(__m512i __W) {
_mm512_reduce_operator_32bit(__W, &, i, i);
}
-static __inline__ int __DEFAULT_FN_ATTRS
+static __inline__ int __DEFAULT_FN_ATTRS
_mm512_reduce_or_epi32(__m512i __W) {
_mm512_reduce_operator_32bit(__W, |, i, i);
}
@@ -10065,7 +10069,7 @@ _mm512_reduce_mul_ps(__m512 __W) {
}
// Vec512 - Vector with size 512.
-// Vec512Neutral - All vector elements set to the identity element.
+// Vec512Neutral - All vector elements set to the identity element.
// Identity element: {+,0},{*,1},{&,0xFFFFFFFF},{|,0}
// Operator - Can be one of following: +,*,&,|
// Mask - Intrinsic Mask
@@ -10095,7 +10099,7 @@ _mm512_mask_reduce_mul_epi32( __mmask16
static __inline__ int __DEFAULT_FN_ATTRS
_mm512_mask_reduce_and_epi32( __mmask16 __M, __m512i __W) {
- _mm512_mask_reduce_operator_32bit(__W, _mm512_set1_epi32(0xFFFFFFFF), &, __M,
+ _mm512_mask_reduce_operator_32bit(__W, _mm512_set1_epi32(0xFFFFFFFF), &, __M,
i, i, d);
}
@@ -10158,7 +10162,7 @@ _mm512_mask_reduce_mul_ps(__mmask16 __M,
return Vec512[0]; \
})
-static __inline__ long long __DEFAULT_FN_ATTRS
+static __inline__ long long __DEFAULT_FN_ATTRS
_mm512_reduce_max_epi64(__m512i __V) {
_mm512_reduce_maxMin_64bit(__V, max_epi64, i, i);
}
@@ -10168,7 +10172,7 @@ _mm512_reduce_max_epu64(__m512i __V) {
_mm512_reduce_maxMin_64bit(__V, max_epu64, i, i);
}
-static __inline__ double __DEFAULT_FN_ATTRS
+static __inline__ double __DEFAULT_FN_ATTRS
_mm512_reduce_max_pd(__m512d __V) {
_mm512_reduce_maxMin_64bit(__V, max_pd, d, f);
}
@@ -10183,7 +10187,7 @@ _mm512_reduce_min_epu64(__m512i __V) {
_mm512_reduce_maxMin_64bit(__V, min_epu64, i, i);
}
-static __inline__ double __DEFAULT_FN_ATTRS
+static __inline__ double __DEFAULT_FN_ATTRS
_mm512_reduce_min_pd(__m512d __V) {
_mm512_reduce_maxMin_64bit(__V, min_pd, d, f);
}
Modified: cfe/branches/release_50/lib/Headers/avxintrin.h
URL: http://llvm.org/viewvc/llvm-project/cfe/branches/release_50/lib/Headers/avxintrin.h?rev=309588&r1=309587&r2=309588&view=diff
==============================================================================
--- cfe/branches/release_50/lib/Headers/avxintrin.h (original)
+++ cfe/branches/release_50/lib/Headers/avxintrin.h Mon Jul 31 10:29:29 2017
@@ -3590,7 +3590,8 @@ _mm_maskstore_ps(float *__p, __m128i __m
static __inline void __DEFAULT_FN_ATTRS
_mm256_stream_si256(__m256i *__a, __m256i __b)
{
- __builtin_nontemporal_store((__v4di)__b, (__v4di*)__a);
+ typedef __v4di __v4di_aligned __attribute__((aligned(32)));
+ __builtin_nontemporal_store((__v4di_aligned)__b, (__v4di_aligned*)__a);
}
/// \brief Moves double-precision values from a 256-bit vector of [4 x double]
@@ -3609,7 +3610,8 @@ _mm256_stream_si256(__m256i *__a, __m256
static __inline void __DEFAULT_FN_ATTRS
_mm256_stream_pd(double *__a, __m256d __b)
{
- __builtin_nontemporal_store((__v4df)__b, (__v4df*)__a);
+ typedef __v4df __v4df_aligned __attribute__((aligned(32)));
+ __builtin_nontemporal_store((__v4df_aligned)__b, (__v4df_aligned*)__a);
}
/// \brief Moves single-precision floating point values from a 256-bit vector
@@ -3629,7 +3631,8 @@ _mm256_stream_pd(double *__a, __m256d __
static __inline void __DEFAULT_FN_ATTRS
_mm256_stream_ps(float *__p, __m256 __a)
{
- __builtin_nontemporal_store((__v8sf)__a, (__v8sf*)__p);
+ typedef __v8sf __v8sf_aligned __attribute__((aligned(32)));
+ __builtin_nontemporal_store((__v8sf_aligned)__a, (__v8sf_aligned*)__p);
}
/* Create vectors */
Copied: cfe/branches/release_50/test/CodeGen/x86-nontemporal.c (from r309382, cfe/trunk/test/CodeGen/x86-nontemporal.c)
URL: http://llvm.org/viewvc/llvm-project/cfe/branches/release_50/test/CodeGen/x86-nontemporal.c?p2=cfe/branches/release_50/test/CodeGen/x86-nontemporal.c&p1=cfe/trunk/test/CodeGen/x86-nontemporal.c&r1=309382&r2=309588&rev=309588&view=diff
==============================================================================
--- cfe/trunk/test/CodeGen/x86-nontemporal.c (original)
+++ cfe/branches/release_50/test/CodeGen/x86-nontemporal.c Mon Jul 31 10:29:29 2017
@@ -1,12 +1,12 @@
// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK
// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -emit-llvm -o - -Wall -Werror -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK16
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK16
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -emit-llvm -o - -Wall -Werror -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK
#include <x86intrin.h>
-// FIXME: (PR33830) Tests showing failure to correctly align non-temporal load/stores on darwin targets where fmax-type-align is set to 16.
+// (PR33830) Tests ensure the correct alignment of non-temporal load/stores on darwin targets where fmax-type-align is set to 16.
//
// 128-bit vectors
@@ -43,36 +43,24 @@ __m128i test_mm_stream_load_si128(__m128
void test_mm256_stream_pd(double* A, __m256d B) {
// CHECK-LABEL: test_mm256_stream_pd
// CHECK: store <4 x double> %{{.*}}, <4 x double>* %{{.*}}, align 32, !nontemporal
-
- // CHECK16-LABEL: test_mm256_stream_pd
- // CHECK16: store <4 x double> %{{.*}}, <4 x double>* %{{.*}}, align 16, !nontemporal
_mm256_stream_pd(A, B);
}
void test_mm256_stream_ps(float* A, __m256 B) {
// CHECK-LABEL: test_mm256_stream_ps
// CHECK: store <8 x float> %{{.*}}, <8 x float>* %{{.*}}, align 32, !nontemporal
-
- // CHECK16-LABEL: test_mm256_stream_ps
- // CHECK16: store <8 x float> %{{.*}}, <8 x float>* %{{.*}}, align 16, !nontemporal
_mm256_stream_ps(A, B);
}
void test_mm256_stream_si256(__m256i* A, __m256i B) {
// CHECK-LABEL: test_mm256_stream_si256
// CHECK: store <4 x i64> %{{.*}}, <4 x i64>* %{{.*}}, align 32, !nontemporal
-
- // CHECK16-LABEL: test_mm256_stream_si256
- // CHECK16: store <4 x i64> %{{.*}}, <4 x i64>* %{{.*}}, align 16, !nontemporal
_mm256_stream_si256(A, B);
}
__m256i test_mm256_stream_load_si256(__m256i const *A) {
// CHECK-LABEL: test_mm256_stream_load_si256
// CHECK: load <4 x i64>, <4 x i64>* %{{.*}}, align 32, !nontemporal
-
- // CHECK16-LABEL: test_mm256_stream_load_si256
- // CHECK16: load <4 x i64>, <4 x i64>* %{{.*}}, align 16, !nontemporal
return _mm256_stream_load_si256(A);
}
@@ -83,35 +71,23 @@ __m256i test_mm256_stream_load_si256(__m
void test_mm512_stream_pd(double* A, __m512d B) {
// CHECK-LABEL: test_mm512_stream_pd
// CHECK: store <8 x double> %{{.*}}, <8 x double>* %{{.*}}, align 64, !nontemporal
-
- // CHECK16-LABEL: test_mm512_stream_pd
- // CHECK16: store <8 x double> %{{.*}}, <8 x double>* %{{.*}}, align 16, !nontemporal
_mm512_stream_pd(A, B);
}
void test_mm512_stream_ps(float* A, __m512 B) {
// CHECK-LABEL: test_mm512_stream_ps
// CHECK: store <16 x float> %{{.*}}, <16 x float>* %{{.*}}, align 64, !nontemporal
-
- // CHECK16-LABEL: test_mm512_stream_ps
- // CHECK16: store <16 x float> %{{.*}}, <16 x float>* %{{.*}}, align 16, !nontemporal
_mm512_stream_ps(A, B);
}
void test_mm512_stream_si512(__m512i* A, __m512i B) {
// CHECK-LABEL: test_mm512_stream_si512
// CHECK: store <8 x i64> %{{.*}}, <8 x i64>* %{{.*}}, align 64, !nontemporal
-
- // CHECK16-LABEL: test_mm512_stream_si512
- // CHECK16: store <8 x i64> %{{.*}}, <8 x i64>* %{{.*}}, align 16, !nontemporal
_mm512_stream_si512(A, B);
}
__m512i test_mm512_stream_load_si512(void *A) {
// CHECK-LABEL: test_mm512_stream_load_si512
// CHECK: load <8 x i64>, <8 x i64>* %{{.*}}, align 64, !nontemporal
-
- // CHECK16-LABEL: test_mm512_stream_load_si512
- // CHECK16: load <8 x i64>, <8 x i64>* %{{.*}}, align 16, !nontemporal
return _mm512_stream_load_si512(A);
}
More information about the llvm-branch-commits
mailing list