diff options
author | Hans Wennborg <hans@hanshq.net> | 2017-07-31 17:29:29 +0000 |
---|---|---|
committer | Hans Wennborg <hans@hanshq.net> | 2017-07-31 17:29:29 +0000 |
commit | 888cdb53e0f71201556f7d19d59c6bc3a582baf2 (patch) | |
tree | 7543977a30ddbc8c3b538074bc457a99a1cb4177 | |
parent | 6f5e1cc9f728bac74f436943ed9dab704c426ef5 (diff) |
Merging r309382:
------------------------------------------------------------------------
r309382 | rksimon | 2017-07-28 06:47:02 -0700 (Fri, 28 Jul 2017) | 3 lines
[X86] Add tests showing inability of vector non-temporal load/store intrinsic to force pointer alignment (PR33830)
Clang specifies a max type alignment of 16 bytes on darwin targets, meaning that the builtin nontemporal stores don't correctly align the loads/stores to 32 or 64 bytes when required, resulting in lowering to temporal unaligned loads/stores.
------------------------------------------------------------------------
Merging r309383:
------------------------------------------------------------------------
r309383 | rksimon | 2017-07-28 07:01:51 -0700 (Fri, 28 Jul 2017) | 1 line
Strip trailing whitespace. NFCI.
------------------------------------------------------------------------
Merging r309488:
------------------------------------------------------------------------
r309488 | rksimon | 2017-07-29 08:33:34 -0700 (Sat, 29 Jul 2017) | 7 lines
[X86][AVX] Ensure vector non-temporal load/store intrinsics force pointer alignment (PR33830)
Clang specifies a max type alignment of 16 bytes on darwin targets (annoyingly in the driver not via cc1), meaning that the builtin nontemporal stores don't correctly align the loads/stores to 32 or 64 bytes when required, resulting in lowering to temporal unaligned loads/stores.
This patch casts the vectors to explicitly aligned types prior to the load/store to ensure that the require alignment is respected.
Differential Revision: https://reviews.llvm.org/D35996
------------------------------------------------------------------------
git-svn-id: https://llvm.org/svn/llvm-project/cfe/branches/release_50@309588 91177308-0d34-0410-b5e6-96231b3b80d8
-rw-r--r-- | lib/Headers/avx2intrin.h | 3 | ||||
-rw-r--r-- | lib/Headers/avx512fintrin.h | 72 | ||||
-rw-r--r-- | lib/Headers/avxintrin.h | 9 | ||||
-rw-r--r-- | test/CodeGen/x86-nontemporal.c | 93 |
4 files changed, 139 insertions, 38 deletions
diff --git a/lib/Headers/avx2intrin.h b/lib/Headers/avx2intrin.h index 5d83a8db48..576f761b25 100644 --- a/lib/Headers/avx2intrin.h +++ b/lib/Headers/avx2intrin.h @@ -832,7 +832,8 @@ _mm256_xor_si256(__m256i __a, __m256i __b) static __inline__ __m256i __DEFAULT_FN_ATTRS _mm256_stream_load_si256(__m256i const *__V) { - return (__m256i)__builtin_nontemporal_load((const __v4di *)__V); + typedef __v4di __v4di_aligned __attribute__((aligned(32))); + return (__m256i)__builtin_nontemporal_load((const __v4di_aligned *)__V); } static __inline__ __m128 __DEFAULT_FN_ATTRS diff --git a/lib/Headers/avx512fintrin.h b/lib/Headers/avx512fintrin.h index b556d04efb..4ce6945311 100644 --- a/lib/Headers/avx512fintrin.h +++ b/lib/Headers/avx512fintrin.h @@ -4289,7 +4289,7 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_cvtps_epu32 ( __mmask16 __U, __m512 __A) { return (__m512i) __builtin_ia32_cvtps2udq512_mask ((__v16sf) __A, - (__v16si) + (__v16si) _mm512_setzero_si512 (), (__mmask16) __U , _MM_FROUND_CUR_DIRECTION); @@ -9035,25 +9035,29 @@ _mm512_kxor (__mmask16 __A, __mmask16 __B) static __inline__ void __DEFAULT_FN_ATTRS _mm512_stream_si512 (__m512i * __P, __m512i __A) { - __builtin_nontemporal_store((__v8di)__A, (__v8di*)__P); + typedef __v8di __v8di_aligned __attribute__((aligned(64))); + __builtin_nontemporal_store((__v8di_aligned)__A, (__v8di_aligned*)__P); } static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_stream_load_si512 (void *__P) { - return (__m512i) __builtin_nontemporal_load((const __v8di *)__P); + typedef __v8di __v8di_aligned __attribute__((aligned(64))); + return (__m512i) __builtin_nontemporal_load((const __v8di_aligned *)__P); } static __inline__ void __DEFAULT_FN_ATTRS _mm512_stream_pd (double *__P, __m512d __A) { - __builtin_nontemporal_store((__v8df)__A, (__v8df*)__P); + typedef __v8df __v8df_aligned __attribute__((aligned(64))); + __builtin_nontemporal_store((__v8df_aligned)__A, (__v8df_aligned*)__P); } static __inline__ void __DEFAULT_FN_ATTRS _mm512_stream_ps (float *__P, __m512 __A) { - __builtin_nontemporal_store((__v16sf)__A, (__v16sf*)__P); + typedef __v16sf __v16sf_aligned __attribute__((aligned(64))); + __builtin_nontemporal_store((__v16sf_aligned)__A, (__v16sf_aligned*)__P); } static __inline__ __m512d __DEFAULT_FN_ATTRS @@ -9217,39 +9221,39 @@ _mm512_maskz_moveldup_ps (__mmask16 __U, __m512 __A) static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_mask_move_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128 __B) { - __m128 res = __A; + __m128 res = __A; res[0] = (__U & 1) ? __B[0] : __W[0]; - return res; + return res; } static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_maskz_move_ss (__mmask8 __U, __m128 __A, __m128 __B) { - __m128 res = __A; - res[0] = (__U & 1) ? __B[0] : 0; - return res; + __m128 res = __A; + res[0] = (__U & 1) ? __B[0] : 0; + return res; } static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_mask_move_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128d __B) { - __m128d res = __A; + __m128d res = __A; res[0] = (__U & 1) ? __B[0] : __W[0]; - return res; + return res; } static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_maskz_move_sd (__mmask8 __U, __m128d __A, __m128d __B) { - __m128d res = __A; - res[0] = (__U & 1) ? __B[0] : 0; - return res; + __m128d res = __A; + res[0] = (__U & 1) ? __B[0] : 0; + return res; } static __inline__ void __DEFAULT_FN_ATTRS _mm_mask_store_ss (float * __W, __mmask8 __U, __m128 __A) { - __builtin_ia32_storess128_mask ((__v16sf *)__W, + __builtin_ia32_storess128_mask ((__v16sf *)__W, (__v16sf) _mm512_castps128_ps512(__A), (__mmask16) __U & (__mmask16)1); } @@ -9257,7 +9261,7 @@ _mm_mask_store_ss (float * __W, __mmask8 __U, __m128 __A) static __inline__ void __DEFAULT_FN_ATTRS _mm_mask_store_sd (double * __W, __mmask8 __U, __m128d __A) { - __builtin_ia32_storesd128_mask ((__v8df *)__W, + __builtin_ia32_storesd128_mask ((__v8df *)__W, (__v8df) _mm512_castpd128_pd512(__A), (__mmask8) __U & 1); } @@ -9606,7 +9610,7 @@ _mm_mask_cvtsd_ss (__m128 __W, __mmask8 __U, __m128 __A, __m128d __B) { return __builtin_ia32_cvtsd2ss_round_mask ((__v4sf)(__A), (__v2df)(__B), - (__v4sf)(__W), + (__v4sf)(__W), (__mmask8)(__U), _MM_FROUND_CUR_DIRECTION); } @@ -9615,7 +9619,7 @@ _mm_maskz_cvtsd_ss (__mmask8 __U, __m128 __A, __m128d __B) { return __builtin_ia32_cvtsd2ss_round_mask ((__v4sf)(__A), (__v2df)(__B), - (__v4sf)_mm_setzero_ps(), + (__v4sf)_mm_setzero_ps(), (__mmask8)(__U), _MM_FROUND_CUR_DIRECTION); } @@ -9680,7 +9684,7 @@ _mm_mask_cvtss_sd (__m128d __W, __mmask8 __U, __m128d __A, __m128 __B) return __builtin_ia32_cvtss2sd_round_mask((__v2df)(__A), (__v4sf)(__B), (__v2df)(__W), - (__mmask8)(__U), _MM_FROUND_CUR_DIRECTION); + (__mmask8)(__U), _MM_FROUND_CUR_DIRECTION); } static __inline__ __m128d __DEFAULT_FN_ATTRS @@ -9688,8 +9692,8 @@ _mm_maskz_cvtss_sd (__mmask8 __U, __m128d __A, __m128 __B) { return __builtin_ia32_cvtss2sd_round_mask((__v2df)(__A), (__v4sf)(__B), - (__v2df)_mm_setzero_pd(), - (__mmask8)(__U), _MM_FROUND_CUR_DIRECTION); + (__v2df)_mm_setzero_pd(), + (__mmask8)(__U), _MM_FROUND_CUR_DIRECTION); } static __inline__ __m128d __DEFAULT_FN_ATTRS @@ -9935,7 +9939,7 @@ static __inline__ double __DEFAULT_FN_ATTRS _mm512_reduce_mul_pd(__m512d __W) { } // Vec512 - Vector with size 512. -// Vec512Neutral - All vector elements set to the identity element. +// Vec512Neutral - All vector elements set to the identity element. // Identity element: {+,0},{*,1},{&,0xFFFFFFFFFFFFFFFF},{|,0} // Operator - Can be one of following: +,*,&,| // Mask - Intrinsic Mask @@ -9965,19 +9969,19 @@ _mm512_mask_reduce_mul_epi64(__mmask8 __M, __m512i __W) { static __inline__ long long __DEFAULT_FN_ATTRS _mm512_mask_reduce_and_epi64(__mmask8 __M, __m512i __W) { - _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_epi64(0xFFFFFFFFFFFFFFFF), + _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_epi64(0xFFFFFFFFFFFFFFFF), &, __M, i, i, q); } static __inline__ long long __DEFAULT_FN_ATTRS _mm512_mask_reduce_or_epi64(__mmask8 __M, __m512i __W) { - _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_epi64(0), |, __M, + _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_epi64(0), |, __M, i, i, q); } static __inline__ double __DEFAULT_FN_ATTRS _mm512_mask_reduce_add_pd(__mmask8 __M, __m512d __W) { - _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_pd(0), +, __M, + _mm512_mask_reduce_operator_64bit(__W, _mm512_set1_pd(0), +, __M, f, d, pd); } @@ -10039,17 +10043,17 @@ _mm512_reduce_add_epi32(__m512i __W) { _mm512_reduce_operator_32bit(__W, +, i, i); } -static __inline__ int __DEFAULT_FN_ATTRS +static __inline__ int __DEFAULT_FN_ATTRS _mm512_reduce_mul_epi32(__m512i __W) { _mm512_reduce_operator_32bit(__W, *, i, i); } -static __inline__ int __DEFAULT_FN_ATTRS +static __inline__ int __DEFAULT_FN_ATTRS _mm512_reduce_and_epi32(__m512i __W) { _mm512_reduce_operator_32bit(__W, &, i, i); } -static __inline__ int __DEFAULT_FN_ATTRS +static __inline__ int __DEFAULT_FN_ATTRS _mm512_reduce_or_epi32(__m512i __W) { _mm512_reduce_operator_32bit(__W, |, i, i); } @@ -10065,7 +10069,7 @@ _mm512_reduce_mul_ps(__m512 __W) { } // Vec512 - Vector with size 512. -// Vec512Neutral - All vector elements set to the identity element. +// Vec512Neutral - All vector elements set to the identity element. // Identity element: {+,0},{*,1},{&,0xFFFFFFFF},{|,0} // Operator - Can be one of following: +,*,&,| // Mask - Intrinsic Mask @@ -10095,7 +10099,7 @@ _mm512_mask_reduce_mul_epi32( __mmask16 __M, __m512i __W) { static __inline__ int __DEFAULT_FN_ATTRS _mm512_mask_reduce_and_epi32( __mmask16 __M, __m512i __W) { - _mm512_mask_reduce_operator_32bit(__W, _mm512_set1_epi32(0xFFFFFFFF), &, __M, + _mm512_mask_reduce_operator_32bit(__W, _mm512_set1_epi32(0xFFFFFFFF), &, __M, i, i, d); } @@ -10158,7 +10162,7 @@ _mm512_mask_reduce_mul_ps(__mmask16 __M, __m512 __W) { return Vec512[0]; \ }) -static __inline__ long long __DEFAULT_FN_ATTRS +static __inline__ long long __DEFAULT_FN_ATTRS _mm512_reduce_max_epi64(__m512i __V) { _mm512_reduce_maxMin_64bit(__V, max_epi64, i, i); } @@ -10168,7 +10172,7 @@ _mm512_reduce_max_epu64(__m512i __V) { _mm512_reduce_maxMin_64bit(__V, max_epu64, i, i); } -static __inline__ double __DEFAULT_FN_ATTRS +static __inline__ double __DEFAULT_FN_ATTRS _mm512_reduce_max_pd(__m512d __V) { _mm512_reduce_maxMin_64bit(__V, max_pd, d, f); } @@ -10183,7 +10187,7 @@ _mm512_reduce_min_epu64(__m512i __V) { _mm512_reduce_maxMin_64bit(__V, min_epu64, i, i); } -static __inline__ double __DEFAULT_FN_ATTRS +static __inline__ double __DEFAULT_FN_ATTRS _mm512_reduce_min_pd(__m512d __V) { _mm512_reduce_maxMin_64bit(__V, min_pd, d, f); } diff --git a/lib/Headers/avxintrin.h b/lib/Headers/avxintrin.h index 78bb70740b..dff5897b6b 100644 --- a/lib/Headers/avxintrin.h +++ b/lib/Headers/avxintrin.h @@ -3590,7 +3590,8 @@ _mm_maskstore_ps(float *__p, __m128i __m, __m128 __a) static __inline void __DEFAULT_FN_ATTRS _mm256_stream_si256(__m256i *__a, __m256i __b) { - __builtin_nontemporal_store((__v4di)__b, (__v4di*)__a); + typedef __v4di __v4di_aligned __attribute__((aligned(32))); + __builtin_nontemporal_store((__v4di_aligned)__b, (__v4di_aligned*)__a); } /// \brief Moves double-precision values from a 256-bit vector of [4 x double] @@ -3609,7 +3610,8 @@ _mm256_stream_si256(__m256i *__a, __m256i __b) static __inline void __DEFAULT_FN_ATTRS _mm256_stream_pd(double *__a, __m256d __b) { - __builtin_nontemporal_store((__v4df)__b, (__v4df*)__a); + typedef __v4df __v4df_aligned __attribute__((aligned(32))); + __builtin_nontemporal_store((__v4df_aligned)__b, (__v4df_aligned*)__a); } /// \brief Moves single-precision floating point values from a 256-bit vector @@ -3629,7 +3631,8 @@ _mm256_stream_pd(double *__a, __m256d __b) static __inline void __DEFAULT_FN_ATTRS _mm256_stream_ps(float *__p, __m256 __a) { - __builtin_nontemporal_store((__v8sf)__a, (__v8sf*)__p); + typedef __v8sf __v8sf_aligned __attribute__((aligned(32))); + __builtin_nontemporal_store((__v8sf_aligned)__a, (__v8sf_aligned*)__p); } /* Create vectors */ diff --git a/test/CodeGen/x86-nontemporal.c b/test/CodeGen/x86-nontemporal.c new file mode 100644 index 0000000000..5e9e42c9f2 --- /dev/null +++ b/test/CodeGen/x86-nontemporal.c @@ -0,0 +1,93 @@ +// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK +// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s --check-prefix=CHECK + +// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -emit-llvm -o - -Wall -Werror -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK +// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +sse4.1 -target-feature +avx -target-feature +avx2 -target-feature +avx512f -fno-signed-char -emit-llvm -o - -Wall -Werror -fmax-type-align=16 | FileCheck %s --check-prefix=CHECK + +#include <x86intrin.h> + +// (PR33830) Tests ensure the correct alignment of non-temporal load/stores on darwin targets where fmax-type-align is set to 16. + +// +// 128-bit vectors +// + +void test_mm_stream_pd(double* A, __m128d B) { + // CHECK-LABEL: test_mm_stream_pd + // CHECK: store <2 x double> %{{.*}}, <2 x double>* %{{.*}}, align 16, !nontemporal + _mm_stream_pd(A, B); +} + +void test_mm_stream_ps(float* A, __m128 B) { + // CHECK16-LABEL: test_mm_stream_ps + // CHECK16: store <4 x float> %{{.*}}, <4 x float>* %{{.*}}, align 16, !nontemporal + _mm_stream_ps(A, B); +} + +void test_mm_stream_si128(__m128i* A, __m128i B) { + // CHECK-LABEL: test_mm_stream_si128 + // CHECK: store <2 x i64> %{{.*}}, <2 x i64>* %{{.*}}, align 16, !nontemporal + _mm_stream_si128(A, B); +} + +__m128i test_mm_stream_load_si128(__m128i const *A) { + // CHECK-LABEL: test_mm_stream_load_si128 + // CHECK: load <2 x i64>, <2 x i64>* %{{.*}}, align 16, !nontemporal + return _mm_stream_load_si128(A); +} + +// +// 256-bit vectors +// + +void test_mm256_stream_pd(double* A, __m256d B) { + // CHECK-LABEL: test_mm256_stream_pd + // CHECK: store <4 x double> %{{.*}}, <4 x double>* %{{.*}}, align 32, !nontemporal + _mm256_stream_pd(A, B); +} + +void test_mm256_stream_ps(float* A, __m256 B) { + // CHECK-LABEL: test_mm256_stream_ps + // CHECK: store <8 x float> %{{.*}}, <8 x float>* %{{.*}}, align 32, !nontemporal + _mm256_stream_ps(A, B); +} + +void test_mm256_stream_si256(__m256i* A, __m256i B) { + // CHECK-LABEL: test_mm256_stream_si256 + // CHECK: store <4 x i64> %{{.*}}, <4 x i64>* %{{.*}}, align 32, !nontemporal + _mm256_stream_si256(A, B); +} + +__m256i test_mm256_stream_load_si256(__m256i const *A) { + // CHECK-LABEL: test_mm256_stream_load_si256 + // CHECK: load <4 x i64>, <4 x i64>* %{{.*}}, align 32, !nontemporal + return _mm256_stream_load_si256(A); +} + +// +// 512-bit vectors +// + +void test_mm512_stream_pd(double* A, __m512d B) { + // CHECK-LABEL: test_mm512_stream_pd + // CHECK: store <8 x double> %{{.*}}, <8 x double>* %{{.*}}, align 64, !nontemporal + _mm512_stream_pd(A, B); +} + +void test_mm512_stream_ps(float* A, __m512 B) { + // CHECK-LABEL: test_mm512_stream_ps + // CHECK: store <16 x float> %{{.*}}, <16 x float>* %{{.*}}, align 64, !nontemporal + _mm512_stream_ps(A, B); +} + +void test_mm512_stream_si512(__m512i* A, __m512i B) { + // CHECK-LABEL: test_mm512_stream_si512 + // CHECK: store <8 x i64> %{{.*}}, <8 x i64>* %{{.*}}, align 64, !nontemporal + _mm512_stream_si512(A, B); +} + +__m512i test_mm512_stream_load_si512(void *A) { + // CHECK-LABEL: test_mm512_stream_load_si512 + // CHECK: load <8 x i64>, <8 x i64>* %{{.*}}, align 64, !nontemporal + return _mm512_stream_load_si512(A); +} |